SmartBench/CMVR-IOT
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 语音交互评估（部分）

Browse Source
This commit is contained in:
stream 2026-01-30 17:27:38 +08:00
parent 56c2d2cf2f
commit 8950f7cd06
20 changed files with 485 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
cmvr-iot-admin/src/main/java/com/cmvr/web/controller/evaluation/AeEvaluationController.java Normal file
View File

@ -0,0 +1,32 @@
package com.cmvr.web.controller.evaluation;
import com.cmvr.common.core.controller.BaseController;
import com.cmvr.common.core.domain.AjaxResult;
import com.cmvr.evaluation.model.domain.AeEvaluation;
import com.cmvr.evaluation.service.IAeEvaluationService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@Api(tags = "AI评估")
@RestController
@RequestMapping("/evaluation")
@RequiredArgsConstructor
public class AeEvaluationController extends BaseController {
private final IAeEvaluationService aeEvaluationService;
@ApiOperation("回调AI评估")
@GetMapping("/test")
public AjaxResult execute() {
AeEvaluation aeEvaluation = new AeEvaluation();
aeEvaluation.setInstId("1336caf1e80b2d23b706ad1cd05d78bd");
aeEvaluation.setItemId("18483d62ccec13c0a336d1d605e9b06c");
aeEvaluationService.executeEvaluation(aeEvaluation);
return AjaxResult.ok();
}
}

2
cmvr-iot-admin/src/main/resources/application-dev.yml
View File

@ -120,6 +120,8 @@ api:
INTENT_RECOGNITION: # 意图识别
app-id: d3c98sp0gdon0fcf6l00
app-key: d3c98vp0gdon0fcf6n20
evaluation: http://192.168.0.18:8000/analyze
flowise:
tts: 192.168.0.222:8080/tts/

2
cmvr-iot-api/cmvr-iot-llm/src/main/java/com/cmvr/llm/config/APIProperties.java
View File

@ -41,6 +41,8 @@ public class APIProperties {
/** 多应用配置（key=ActionEnum) */
private Map<String, AgentConfig> agents;
private String evaluation;
/**
* 根据 ActionEnum 名称获取 app 配置
*/

34
cmvr-iot-evaluation/pom.xml Normal file
View File

@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.cmvr</groupId>
<artifactId>cmvr-iot</artifactId>
<version>3.8.9</version>
</parent>
<artifactId>cmvr-iot-evaluation</artifactId>
<description>
AI评估模块
</description>
<dependencies>
<!-- 通用工具-->
<dependency>
<groupId>com.cmvr</groupId>
<artifactId>cmvr-iot-common</artifactId>
</dependency>
<!-- 测试流程-->
<dependency>
<groupId>com.cmvr</groupId>
<artifactId>cmvr-iot-test</artifactId>
</dependency>
</dependencies>
</project>

7
cmvr-iot-evaluation/src/main/java/com/cmvr/evaluation/mapper/AeEvaluationMapper.java Normal file
View File

@ -0,0 +1,7 @@
package com.cmvr.evaluation.mapper;
import com.cmvr.evaluation.model.domain.AeEvaluation;
import com.github.yulichang.base.MPJBaseMapper;
public interface AeEvaluationMapper extends MPJBaseMapper<AeEvaluation> {
}

45
cmvr-iot-evaluation/src/main/java/com/cmvr/evaluation/model/domain/AeEvaluation.java Normal file
View File

@ -0,0 +1,45 @@
package com.cmvr.evaluation.model.domain;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.io.Serializable;
/**
* AI评估实体类
*/
@Data
public class AeEvaluation implements Serializable {
private static final long serialVersionUID = 1L;
@ApiModelProperty("主键id")
@TableId(value = "ae_id", type = IdType.ASSIGN_UUID)
private String aeId;
@ApiModelProperty("评估类型（VOICE:语音，TOUCH:触控）")
private String aeType;
@ApiModelProperty("流程实例ID")
private String instId;
@ApiModelProperty("检测项ID")
private String itemId;
@ApiModelProperty("音视频内容")
private String avContent;
@ApiModelProperty("音频地址")
private String audioPath;
@ApiModelProperty("视频地址")
private String videoPath;
@ApiModelProperty("评估状态，0:待评估，1:评估中，3:评估成功，4:评估失败")
private Integer status;
@ApiModelProperty("评估最终结果（JSON）")
private String result;
}

12
cmvr-iot-evaluation/src/main/java/com/cmvr/evaluation/service/IAeEvaluationService.java Normal file
View File

@ -0,0 +1,12 @@
package com.cmvr.evaluation.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.cmvr.evaluation.model.domain.AeEvaluation;
public interface IAeEvaluationService extends IService<AeEvaluation> {
/**
* 执行语音交互评估
*/
public void executeEvaluation(AeEvaluation evaluation);
}

21
cmvr-iot-evaluation/src/main/java/com/cmvr/evaluation/service/ex/ExAeEvaluationServiceImpl.java Normal file
View File

@ -0,0 +1,21 @@
package com.cmvr.evaluation.service.ex;
import com.alibaba.fastjson2.JSONObject;
import com.cmvr.evaluation.model.domain.AeEvaluation;
import com.cmvr.evaluation.service.IAeEvaluationService;
import com.cmvr.test.service.ex.ExAeEvaluationService;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
@Service
@RequiredArgsConstructor
public class ExAeEvaluationServiceImpl implements ExAeEvaluationService {
private final IAeEvaluationService aeEvaluationService;
@Override
public void executeEvaluation(JSONObject body) {
AeEvaluation aeEvaluation = body.toJavaObject(AeEvaluation.class);
aeEvaluationService.executeEvaluation(aeEvaluation);
}
}

253
cmvr-iot-evaluation/src/main/java/com/cmvr/evaluation/service/impl/AeEvaluationServiceImpl.java Normal file
View File

@ -0,0 +1,253 @@
package com.cmvr.evaluation.service.impl;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.cmvr.common.utils.http.CallAPIUtil;
import com.cmvr.evaluation.mapper.AeEvaluationMapper;
import com.cmvr.evaluation.model.domain.AeEvaluation;
import com.cmvr.evaluation.service.IAeEvaluationService;
import com.cmvr.llm.config.APIProperties;
import com.cmvr.test.enums.ActionEnum;
import com.cmvr.test.model.domain.TeNodeInst;
import com.cmvr.test.model.vo.TeFlowViewVO;
import com.cmvr.test.service.ITeNodeInstService;
import lombok.RequiredArgsConstructor;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
public class AeEvaluationServiceImpl extends ServiceImpl<AeEvaluationMapper, AeEvaluation> implements IAeEvaluationService {
private final ITeNodeInstService teNodeInstService;
private final APIProperties apiProperties;
private final StringRedisTemplate redisTemplate;
public void executeEvaluation(AeEvaluation evaluation) {
// 查询流程执行记录
TeFlowViewVO teFlowViewVO = new TeFlowViewVO();
teFlowViewVO.setInstId(evaluation.getInstId());
teFlowViewVO.setItemId(evaluation.getItemId());
List<TeNodeInst> view = teNodeInstService.view(teFlowViewVO);
// 解析 VI_SCHEME 参数
TeNodeInst viSchemeInst = view.stream()
.filter(e -> ActionEnum.VI_SCHEME.getAction().equals(e.getAction()))
.findFirst()
.orElseThrow(() -> new IllegalStateException("未找到 VI_SCHEME 节点"));
JSONObject paramsOut = JSONObject.parseObject(viSchemeInst.getParamsOut());
Integer sceneType = paramsOut.getInteger("sceneType");
// 唤醒语料
JSONObject wakeCorpus = paramsOut.getJSONObject("wakeCorpus");
String wakeContent = wakeCorpus != null ? wakeCorpus.getString("textContent") : null;
// 测试语料
JSONArray testCorpusArray = paramsOut.getJSONArray("testCorpus");
// 构建评估任务列表
List<AeEvaluation> evaluations = new LinkedList<>();
switch (sceneType) {
// 场景 1：唤醒
case 1: {
AeEvaluation eval = new AeEvaluation();
BeanUtil.copyProperties(evaluation, eval);
eval.setAeId(IdUtil.fastSimpleUUID());
eval.setStatus(0); // 待评估
eval.setAvContent(wakeContent);
evaluations.add(eval);
break;
}
// 场景 2：单次对话（一层循环）
case 2: {
Map<String, List<TeNodeInst>> iterationMap = buildSingleIterationMap(view);
for (Map.Entry<String, List<TeNodeInst>> entry : iterationMap.entrySet()) {
String iterationKey = entry.getKey(); // "1" / "2" / ...
List<TeNodeInst> nodeList = entry.getValue();
int index = Integer.parseInt(iterationKey) - 1;
JSONObject corpus = testCorpusArray.getJSONObject(index);
String content = StrUtil.join(", ",
wakeContent,
corpus.getString("textContent")
);
String audioUrl = extractAudioUrl(nodeList);
String videoUrl = extractVideoUrl(nodeList);
AeEvaluation eval = new AeEvaluation();
BeanUtil.copyProperties(evaluation, eval);
eval.setAeId(IdUtil.fastSimpleUUID());
eval.setStatus(0);
eval.setAvContent(content);
eval.setAudioPath(audioUrl);
eval.setVideoPath(videoUrl);
evaluations.add(eval);
}
break;
}
// 场景 3：连续对话（嵌套循环）
case 3: {
Map<String, List<TeNodeInst>> ContinuousIterationMap = buildContinuousIterationMap(view);
for (Map.Entry<String, List<TeNodeInst>> entry : ContinuousIterationMap.entrySet()) {
String key = entry.getKey();
List<TeNodeInst> value = entry.getValue();
// 每一个 outer iteration 代表一次完整连续对话
StringBuilder contentBuilder = new StringBuilder(wakeContent);
int corpusIndex = Integer.parseInt(key) - 1;
JSONArray corpus = testCorpusArray.getJSONArray(corpusIndex);
String corpusText = corpus.stream()
.filter(Objects::nonNull)
.map(obj -> ((JSONObject) obj).getString("textContent"))
.filter(StrUtil::isNotEmpty)
.collect(Collectors.joining(", "));
if (StrUtil.isNotEmpty(corpusText)) {
contentBuilder.append(", ").append(corpusText);
}
String audioUrl = extractAudioUrl(value);
String videoUrl = extractVideoUrl(value);
AeEvaluation eval = new AeEvaluation();
BeanUtil.copyProperties(evaluation, eval);
eval.setAeId(IdUtil.fastSimpleUUID());
eval.setStatus(0);
eval.setAvContent(contentBuilder.toString());
eval.setAudioPath(audioUrl);
eval.setVideoPath(videoUrl);
evaluations.add(eval);
}
break;
}
default:
throw new IllegalArgumentException("未知 sceneType：" + sceneType);
}
try {
// 入库
this.saveBatch(evaluations);
// 提取第一个对象出来
AeEvaluation first = CollUtil.getFirst(evaluations);
List<AeEvaluation> rest = CollUtil.sub(evaluations, 1, evaluations.size());
// 剩下的先入队列
for (AeEvaluation e : rest) {
redisTemplate.opsForList().rightPush("ae:evaluation:queue", JSON.toJSONString(e));
}
// 构建评估请求
Map<String, String> body = new HashMap<>();
body.put("aeId", first.getAeId());
body.put("audioPath", first.getAudioPath());
body.put("videoPath", first.getVideoPath());
body.put("content", first.getAvContent());
body.put("aeType", first.getAeType());
CallAPIUtil.doPostJson(apiProperties.getEvaluation(), null, body);
first.setStatus(1);
this.updateById(first);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private Map<String, List<TeNodeInst>> buildSingleIterationMap(List<TeNodeInst> view) {
return view.stream()
.filter(e -> StrUtil.isNotEmpty(e.getIteration()))
.collect(Collectors.groupingBy(
TeNodeInst::getIteration,
LinkedHashMap::new,
Collectors.collectingAndThen(
Collectors.toList(),
list -> list.stream()
.sorted(Comparator.comparing(TeNodeInst::getId))
.collect(Collectors.toList())
)
));
}
private Map<String, List<TeNodeInst>> buildContinuousIterationMap(List<TeNodeInst> view) {
return view.stream()
.filter(e -> StrUtil.isNotEmpty(e.getIteration()))
// 只保留最外层 iteration（不包含子层）
.filter(e -> !e.getIteration().contains("-"))
.collect(Collectors.groupingBy(
TeNodeInst::getIteration,
LinkedHashMap::new,
Collectors.collectingAndThen(
Collectors.toList(),
list -> list.stream()
.sorted(Comparator.comparing(TeNodeInst::getId))
.collect(Collectors.toList())
)
));
}
private String extractAudioUrl(List<TeNodeInst> nodeList) {
TeNodeInst microPhoneInst = nodeList.stream()
.filter(e -> ActionEnum.MICROPHONE_START.getAction().equals(e.getAction()))
.findFirst()
.orElse(null);
if (microPhoneInst == null) {
return null;
}
JSONObject paramsOut = JSONObject.parseObject(microPhoneInst.getParamsOut());
return paramsOut.getString("audioUrl");
}
private String extractVideoUrl(List<TeNodeInst> nodeList) {
TeNodeInst recordingInst = nodeList.stream()
.filter(e -> ActionEnum.CAMERA_RECORDING_START.getAction().equals(e.getAction()))
.findFirst()
.orElse(null);
if (recordingInst == null) {
return null;
}
JSONObject paramsOut = JSONObject.parseObject(recordingInst.getParamsOut());
return paramsOut.getString("videoUrl");
}
}

3
cmvr-iot-test/src/main/java/com/cmvr/test/enums/RunModeEnum.java
View File

@ -3,5 +3,6 @@ package com.cmvr.test.enums;
public enum RunModeEnum {
NORMAL, //正常
TRIAL, //试运行
PROJECT //项目
VI_PROJECT, //语音交互项目
TI_PROJECT //触控交互项目
}

35
cmvr-iot-test/src/main/java/com/cmvr/test/flow/runtime/dispatcher/FlowEndNodeHandler.java
View File

@ -1,18 +1,53 @@
package com.cmvr.test.flow.runtime.dispatcher;
import com.alibaba.fastjson2.JSONObject;
import com.cmvr.test.enums.RunModeEnum;
import com.cmvr.test.flow.context.TaskContext;
import com.cmvr.test.flow.context.TaskInstHolder;
import com.cmvr.test.flow.runtime.message.TaskNodeExecuteMessage;
import com.cmvr.test.flow.runtime.message.TaskNodeExecuteResult;
import com.cmvr.test.service.ex.ExAeEvaluationService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
@Slf4j
@Component("END")
@RequiredArgsConstructor
public class FlowEndNodeHandler implements FlowNodeTypeHandler {
private final TaskInstHolder taskInstHolder;
private final ExAeEvaluationService exAeEvaluationService;
@Resource(name = "threadPoolTaskExecutor")
private final ThreadPoolTaskExecutor executor;
@Override
public TaskNodeExecuteResult handle(TaskNodeExecuteMessage message) {
String instId = message.getInstId();
String itemId = message.getItemId();
TaskContext context = taskInstHolder.getContext(instId);
RunModeEnum runMode = context.getRunMode();
JSONObject jsonObject = new JSONObject();
jsonObject.put("instId", instId);
jsonObject.put("itemId", itemId);
jsonObject.put("aeType", "VOICE");
executor.submit(() -> {
Thread.currentThread().setName("evaluation-thread-" + Thread.currentThread().getId());
try {
// log.info("异步任务开始执行：threadName={}, instId={}, taskId={}", Thread.currentThread().getName(), instId, taskId);
exAeEvaluationService.executeEvaluation(jsonObject);
// log.info("异步任务执行结束：threadName={}, instId={}, taskId={}", Thread.currentThread().getName(), instId, taskId);
} catch (Exception e) {
// log.error("任务执行异常：threadName={}, instId={}, taskId={}, 错误={}", Thread.currentThread().getName(), instId, taskId, e.getMessage(), e);
Thread.currentThread().interrupt(); // 保持中断状态
}
});
return TaskNodeExecuteResult.success(message.getInputParams());
}
}

2
cmvr-iot-test/src/main/java/com/cmvr/test/flow/runtime/engine/FlowTaskEngine.java
View File

@ -42,7 +42,7 @@ public class FlowTaskEngine {
message.setTaskId(taskId);
message.setTerminalId(terminalId);
message.setItemId(itemId);
if (runMode.equals(RunModeEnum.PROJECT)) {
if (runMode.equals(RunModeEnum.VI_PROJECT)) {
message.setLoopDetail(item.getSchemeInfo());
}

2
cmvr-iot-test/src/main/java/com/cmvr/test/flow/runtime/engine/FlowTaskRuntimeEntry.java
View File

@ -111,7 +111,7 @@ public class FlowTaskRuntimeEntry implements FlowTaskRuntimeService {
return executeTaskInternal(
taskExecuteProjectVO.getTerminalId(),
taskExecuteProjectVO.getProjectId(),
RunModeEnum.PROJECT,
RunModeEnum.VI_PROJECT,
taskExecuteProjectVO.getRunParams(),
taskExecuteProjectVO
);

2
cmvr-iot-test/src/main/java/com/cmvr/test/flow/runtime/operator/edge/EdgeCameraOperateService.java
View File

@ -124,7 +124,7 @@ public class EdgeCameraOperateService implements EdgeOperateService {
}
case CAMERA_RECORDING_STOP: {
edgeCameraService.stopRecording(edgeCommonVO);
// edgeCameraService.stopRecording(edgeCommonVO);
// todo 结束录像获取视频路径输出
// 获取视频录制开始节点的id
FlowNodeWrapper startRecordNode = message.getGraph()

5
cmvr-iot-test/src/main/java/com/cmvr/test/flow/runtime/operator/edge/EdgeMicrophoneOperateService.java
View File

@ -36,13 +36,14 @@ public class EdgeMicrophoneOperateService implements EdgeOperateService {
edgeCommonVO.setTerminalId(terminalId);
switch (action) {
case MICROPHONE_START: {
String audioPath = edgeMicrophoneService.startRecord(edgeCommonVO);
// String audioPath = edgeMicrophoneService.startRecord(edgeCommonVO);
String audioPath = "xxx";
output.put("audioUrl", audioPath);
break;
}
case MICROPHONE_STOP: {
edgeMicrophoneService.stopRecord(edgeCommonVO);
// edgeMicrophoneService.stopRecord(edgeCommonVO);
break;
}

6
cmvr-iot-test/src/main/java/com/cmvr/test/flow/runtime/operator/edge/vi/ViPlayOperateService.java
View File

@ -57,7 +57,9 @@ public class ViPlayOperateService implements EdgeOperateService {
edgeCommonVO.setTerminalId(terminalId);
}
try {
log.info("语料播放成功，audioPath={}", audioPath);
/*try {
// 播放音频
edgeSpeakerService.playAudio(terminalId, deviceId, audioPath);
@ -92,7 +94,7 @@ public class ViPlayOperateService implements EdgeOperateService {
if (needBioSpeak) {
edgeBioHeadService.speakStop(edgeCommonVO);
}
}
}*/
return TaskNodeExecuteResult.success();
}

10
cmvr-iot-test/src/main/java/com/cmvr/test/service/ex/ExAeEvaluationService.java Normal file
View File

@ -0,0 +1,10 @@
package com.cmvr.test.service.ex;
import com.alibaba.fastjson2.JSONObject;
/**
* 外部-评估
*/
public interface ExAeEvaluationService {
public void executeEvaluation(JSONObject body);
}

6
cmvr-iot-vi/pom.xml
View File

@ -29,6 +29,12 @@
<artifactId>cmvr-iot-test</artifactId>
</dependency>
<!-- AI评估-->
<dependency>
<groupId>com.cmvr</groupId>
<artifactId>cmvr-iot-evaluation</artifactId>
</dependency>
</dependencies>
</project>

8
cmvr-iot-vi/src/main/java/com/cmvr/vi/service/impl/ViProjectServiceImpl.java
View File

@ -7,6 +7,7 @@ import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.cmvr.common.exception.GlobalException;
import com.cmvr.evaluation.model.domain.AeEvaluation;
import com.cmvr.test.enums.FlowSceneTypeEnum;
import com.cmvr.test.enums.RunModeEnum;
import com.cmvr.test.flow.runtime.engine.FlowTaskRuntimeService;
@ -63,10 +64,10 @@ public class ViProjectServiceImpl extends ServiceImpl<ViProjectMapper, ViProject
public int insertViProject(ViProject viProject) {
int insert = this.baseMapper.insert(viProject);
TeTaskConfigInfo taskConfigInfo = new TeTaskConfigInfo();
taskConfigInfo.setRunMode(RunModeEnum.PROJECT.name());
taskConfigInfo.setRunMode(RunModeEnum.VI_PROJECT.name());
taskConfigInfo.setId(viProject.getProjectId());
taskConfigInfo.setStatus("0");
taskConfigInfo.setTaskName(StrUtil.format("{}_{}", RunModeEnum.PROJECT, viProject.getProjectName()));
taskConfigInfo.setTaskName(StrUtil.format("{}_{}", RunModeEnum.VI_PROJECT, viProject.getProjectName()));
taskConfigInfoService.save(taskConfigInfo);
return insert;
}
@ -128,6 +129,9 @@ public class ViProjectServiceImpl extends ServiceImpl<ViProjectMapper, ViProject
.eq(ViProject::getProjectId, taskExecuteProjectVO.getProjectId())
.set(ViProject::getStatus, "0")
);
// 异步执行评估
AeEvaluation aeEvaluation = new AeEvaluation();
aeEvaluation.setInstId(instId);
return instId;
}

8
pom.xml
View File

@ -274,6 +274,13 @@
<version>${cmvr-iot.version}</version>
</dependency>
<!-- AIp评估-->
<dependency>
<groupId>com.cmvr</groupId>
<artifactId>cmvr-iot-evaluation</artifactId>
<version>${cmvr-iot.version}</version>
</dependency>
<!-- 大模型调用-->
<dependency>
<groupId>com.cmvr</groupId>
@ -330,6 +337,7 @@
<module>cmvr-iot-api</module>
<module>cmvr-iot-vi</module>
<module>cmvr-iot-ti</module>
<module>cmvr-iot-evaluation</module>
</modules>
<packaging>pom</packaging>