SmartBench/CMVR-IOT
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into dev

Browse Source
This commit is contained in:
lixiaolong 2025-09-05 15:14:41 +08:00
commit 22a58227e0
24 changed files with 2875 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmvr-iot-admin/pom.xml
View File

@ -60,6 +60,12 @@
<artifactId>cmvr-iot-device</artifactId>
</dependency>
<!-- 语音交互-->
<dependency>
<groupId>com.cmvr</groupId>
<artifactId>cmvr-iot-vi</artifactId>
</dependency>
</dependencies>
<properties>
<env>dev</env>

43
cmvr-iot-admin/src/main/java/com/cmvr/web/controller/TestController.java Normal file
View File

@ -0,0 +1,43 @@
package com.cmvr.web.controller;
import com.cmvr.common.core.controller.BaseController;
import com.cmvr.common.core.domain.AjaxResult;
import com.cmvr.edge.client.service.EdgeHlcService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.io.IOException;
@Api(tags = "通用--测试")
@RestController
@RequestMapping("/common/test")
@RequiredArgsConstructor
public class TestController extends BaseController {
private final EdgeHlcService edgeHlcService;
@ApiOperation("图片标注")
@PostMapping("/mark")
public AjaxResult mark(
@RequestPart("image") MultipartFile image,
@RequestParam("x") Integer x,
@RequestParam("y") Integer y
) {
try {
File tempFile = File.createTempFile("upload_", "_" + image.getOriginalFilename());
image.transferTo(tempFile);
edgeHlcService.markPoint(tempFile,x,y);
} catch (IOException e) {
throw new RuntimeException(e);
}
return AjaxResult.ok();
}
}

99
cmvr-iot-admin/src/main/java/com/cmvr/web/controller/vi/ViCorpusController.java Normal file
View File

@ -0,0 +1,99 @@
package com.cmvr.web.controller.vi;
import com.cmvr.common.core.controller.BaseController;
import com.cmvr.common.core.domain.AjaxResult;
import com.cmvr.common.core.page.TableDataInfo;
import com.cmvr.vi.model.domain.ViCorpus;
import com.cmvr.vi.service.IViCorpusService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.RequiredArgsConstructor;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
/**
* 语料库Controller
*
* @author cmvr-iot
*/
@Api(tags = "语音交互--语料库")
@RestController
@RequestMapping("/vi/corpus")
@RequiredArgsConstructor
public class ViCorpusController extends BaseController {
private final IViCorpusService viCorpusService;
/**
* 查询语料库列表
*/
@ApiOperation("查询语料库列表")
@PreAuthorize("@ss.hasPermi('vi:corpus:list')")
@GetMapping("/list")
public TableDataInfo list(ViCorpus viCorpus) {
startPage();
List<ViCorpus> list = viCorpusService.selectViCorpusList(viCorpus);
return getDataTable(list);
}
// /**
// * 导出语料库列表
// */
// @ApiOperation("查询语料库列表")
// @PreAuthorize("@ss.hasPermi('vi:corpus:export')")
// @PostMapping("/export")
// public void export(HttpServletResponse response, ViCorpus viCorpus) {
// List<ViCorpus> list = viCorpusService.selectViCorpusList(viCorpus);
// ExcelUtil<ViCorpus> util = new ExcelUtil<ViCorpus>(ViCorpus.class);
// util.exportExcel(response, list, "语料库数据");
// }
/**
* 获取语料库详细信息
*/
@ApiOperation("根据id获取语料库详细信息")
@PreAuthorize("@ss.hasPermi('vi:corpus:query')")
@GetMapping(value = "/{corpusId}")
public AjaxResult getInfo(@PathVariable("corpusId") Long corpusId) {
return success(viCorpusService.selectViCorpusByCorpusId(corpusId));
}
/**
* 新增语料库
*/
@ApiOperation("新增语料库")
@PreAuthorize("@ss.hasPermi('vi:corpus:add')")
@PostMapping
public AjaxResult add(@RequestBody ViCorpus viCorpus) {
return toAjax(viCorpusService.insertViCorpus(viCorpus));
}
/**
* 修改语料库
*/
@ApiOperation("修改语料库")
@PreAuthorize("@ss.hasPermi('vi:corpus:edit')")
@PutMapping
public AjaxResult edit(@RequestBody ViCorpus viCorpus) {
return toAjax(viCorpusService.updateViCorpus(viCorpus));
}
/**
* 删除语料库
*/
@ApiOperation("删除语料库")
@PreAuthorize("@ss.hasPermi('vi:corpus:remove')")
@DeleteMapping("/{corpusIds}")
public AjaxResult remove(@PathVariable Long[] corpusIds) {
return toAjax(viCorpusService.deleteViCorpusByCorpusIds(corpusIds));
}
}

2
cmvr-iot-api/cmvr-iot-edge/cmvr-iot-grpc-client/src/main/java/com/cmvr/edge/client/manage/GrpcServiceManager.java
View File

@ -56,6 +56,7 @@ public class GrpcServiceManager {
clientFactories.put(DexHandServiceGrpc.DexHandServiceStub.class, new GrpcClientFactory<>(DexHandServiceGrpc::newStub));
// 注册机械臂服务的stub
clientFactories.put(HumanoidRobotServiceGrpc.HumanoidRobotServiceBlockingStub.class, new GrpcClientFactory<>(HumanoidRobotServiceGrpc::newBlockingStub));
clientFactories.put(HlcServiceGrpc.HlcServiceBlockingStub.class, new GrpcClientFactory<>(HlcServiceGrpc::newBlockingStub));
}
@ -81,6 +82,7 @@ public class GrpcServiceManager {
String grpcAddress = getGrpcServiceAddress(terminalId);
return ManagedChannelBuilder.forTarget(grpcAddress)
.usePlaintext()
.maxInboundMessageSize(50 * 1024 * 1024) // 50MB
.build();
}

16
cmvr-iot-api/cmvr-iot-edge/cmvr-iot-grpc-client/src/main/java/com/cmvr/edge/client/service/EdgeHlcService.java Normal file
View File

@ -0,0 +1,16 @@
package com.cmvr.edge.client.service;
import java.io.File;
public interface EdgeHlcService {
/**
* 触控
*/
public String touch(File image, String terminalId, String deviceId, int x, int y, double maxForce);
/**
* 图片标注
*/
public void markPoint(File image, int x, int y);
}

20
cmvr-iot-api/cmvr-iot-edge/cmvr-iot-grpc-client/src/main/java/com/cmvr/edge/client/service/impl/EdgeCameraServiceImpl.java
View File

@ -161,29 +161,29 @@ public class EdgeCameraServiceImpl implements EdgeCameraService, EdgeStreamServi
// 彩色图和深度图
CameraCommand.FrameData frame = feedback.getColorFrame();
CameraCommand.FrameData depthFrame = feedback.getDepthFrame();
// CameraCommand.FrameData depthFrame = feedback.getDepthFrame();
if (!feedback.hasColorFrame() || frame.getWidth() <= 0 || frame.getHeight() <= 0) {
throw new GlobalException("响应图片数据无效");
}
String filePath = saveImage(frame.getData(), frame.getWidth(), frame.getHeight(), false);
String deepFilePath = saveImage(depthFrame.getData(), depthFrame.getWidth(), depthFrame.getHeight(), true);
List<String> result = new ArrayList<>();
// String deepFilePath = saveImage(depthFrame.getData(), depthFrame.getWidth(), depthFrame.getHeight(), true);
// List<String> result = new ArrayList<>();
try {
MultipartFile file = convertFileToMultipartFile(filePath);
MultipartFile deepFile = convertFileToMultipartFile(deepFilePath);
String imageUrl = sysFileService.uploadFile(file, FileType.IMAGE.code());
result.add(imageUrl);
String deepImageUrl = sysFileService.uploadFile(deepFile, FileType.IMAGE.code());
result.add(deepImageUrl);
// MultipartFile deepFile = convertFileToMultipartFile(deepFilePath);
return sysFileService.uploadFile(file, FileType.IMAGE.code());
// result.add(imageUrl);
// String deepImageUrl = sysFileService.uploadFile(deepFile, FileType.IMAGE.code());
// result.add(deepImageUrl);
} catch (IOException e) {
throw new GlobalException("图片上传失败: " + e.getMessage());
} finally {
deleteTempFile(filePath);
deleteTempFile(deepFilePath);
// deleteTempFile(deepFilePath);
// todo 暂时不关闭摄像头
// stop(terminalId, deviceId);
}
return JSON.toJSONString(result);
// return JSON.toJSONString(result);
}
// 缓冲区和收集状态

84
cmvr-iot-api/cmvr-iot-edge/cmvr-iot-grpc-client/src/main/java/com/cmvr/edge/client/service/impl/EdgeHlcServiceImpl.java Normal file
View File

@ -0,0 +1,84 @@
package com.cmvr.edge.client.service.impl;
import cmvr.api.HlcCommand;
import cmvr.api.HlcServiceGrpc;
import com.alibaba.fastjson2.JSON;
import com.cmvr.edge.client.manage.GrpcServiceManager;
import com.cmvr.edge.client.service.EdgeHlcService;
import com.cmvr.edge.client.utils.EdgeCommonUtil;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import javax.imageio.ImageIO;
import java.awt.*;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
@Slf4j
@Service
@RequiredArgsConstructor
public class EdgeHlcServiceImpl implements EdgeHlcService {
private final GrpcServiceManager grpcServiceManager;
@Override
public String touch(File image, String terminalId, String deviceId, int x, int y, double maxForce) {
HlcServiceGrpc.HlcServiceBlockingStub stub = grpcServiceManager.getGrpcClient(terminalId, HlcServiceGrpc.HlcServiceBlockingStub.class);
HlcCommand.Touch.Request request = HlcCommand.Touch.Request.newBuilder()
.setHeader(EdgeCommonUtil.buildRequest(deviceId))
.setU(x)
.setV(y)
.setMaxForce(maxForce)
.build();
String response = JSON.toJSONString(stub.touch(request).getHeader());
log.info("触控调用成功，响应结果:{}", response);
markPoint(image,x,y);
return response;
}
/**
* 在图片中指定坐标点进行标注
*
* @param imageFile 原始图片文件
* @param x 坐标 X
* @param y 坐标 Y
*/
@Override
public void markPoint(File imageFile, int x, int y) {
// 读入图片
BufferedImage image;
try {
image = ImageIO.read(imageFile);
} catch (IOException e) {
throw new RuntimeException(e);
}
Graphics2D g = image.createGraphics();
// 抗锯齿
g.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
// 设置画笔颜色
g.setColor(Color.GREEN);
// 半径
int radius = 5;
// 画实心圆
g.fillOval(x - radius, y - radius, radius * 2, radius * 2);
g.dispose();
// 输出文件
String outPath = imageFile.getParent() + File.separator +
"marked_" + imageFile.getName();
File outFile = new File(outPath);
try {
ImageIO.write(image, "png", outFile);
} catch (IOException e) {
throw new RuntimeException(e);
}
log.info("图片标注成功:{}", outPath);
}
}

1942
cmvr-iot-api/cmvr-iot-edge/cmvr-iot-grpc-lib/src/main/java/cmvr/api/HlcCommand.java Normal file
View File

File diff suppressed because it is too large Load Diff

276
cmvr-iot-api/cmvr-iot-edge/cmvr-iot-grpc-lib/src/main/java/cmvr/api/HlcServiceGrpc.java Normal file
View File

@ -0,0 +1,276 @@
package cmvr.api;
import static io.grpc.MethodDescriptor.generateFullMethodName;
/**
*/
@javax.annotation.Generated(
value = "by gRPC proto compiler (version 1.52.0)",
comments = "Source: cmvr/api/hlc_service.proto")
@io.grpc.stub.annotations.GrpcGenerated
public final class HlcServiceGrpc {
private HlcServiceGrpc() {}
public static final String SERVICE_NAME = "cmvr.api.HlcService";
// Static method descriptors that strictly reflect the proto.
private static volatile io.grpc.MethodDescriptor<cmvr.api.HlcCommand.Touch.Request,
cmvr.api.HlcCommand.Touch.Response> getTouchMethod;
@io.grpc.stub.annotations.RpcMethod(
fullMethodName = SERVICE_NAME + '/' + "touch",
requestType = cmvr.api.HlcCommand.Touch.Request.class,
responseType = cmvr.api.HlcCommand.Touch.Response.class,
methodType = io.grpc.MethodDescriptor.MethodType.UNARY)
public static io.grpc.MethodDescriptor<cmvr.api.HlcCommand.Touch.Request,
cmvr.api.HlcCommand.Touch.Response> getTouchMethod() {
io.grpc.MethodDescriptor<cmvr.api.HlcCommand.Touch.Request, cmvr.api.HlcCommand.Touch.Response> getTouchMethod;
if ((getTouchMethod = HlcServiceGrpc.getTouchMethod) == null) {
synchronized (HlcServiceGrpc.class) {
if ((getTouchMethod = HlcServiceGrpc.getTouchMethod) == null) {
HlcServiceGrpc.getTouchMethod = getTouchMethod =
io.grpc.MethodDescriptor.<cmvr.api.HlcCommand.Touch.Request, cmvr.api.HlcCommand.Touch.Response>newBuilder()
.setType(io.grpc.MethodDescriptor.MethodType.UNARY)
.setFullMethodName(generateFullMethodName(SERVICE_NAME, "touch"))
.setSampledToLocalTracing(true)
.setRequestMarshaller(io.grpc.protobuf.ProtoUtils.marshaller(
cmvr.api.HlcCommand.Touch.Request.getDefaultInstance()))
.setResponseMarshaller(io.grpc.protobuf.ProtoUtils.marshaller(
cmvr.api.HlcCommand.Touch.Response.getDefaultInstance()))
.setSchemaDescriptor(new HlcServiceMethodDescriptorSupplier("touch"))
.build();
}
}
}
return getTouchMethod;
}
/**
* Creates a new async stub that supports all call types for the service
*/
public static HlcServiceStub newStub(io.grpc.Channel channel) {
io.grpc.stub.AbstractStub.StubFactory<HlcServiceStub> factory =
new io.grpc.stub.AbstractStub.StubFactory<HlcServiceStub>() {
@java.lang.Override
public HlcServiceStub newStub(io.grpc.Channel channel, io.grpc.CallOptions callOptions) {
return new HlcServiceStub(channel, callOptions);
}
};
return HlcServiceStub.newStub(factory, channel);
}
/**
* Creates a new blocking-style stub that supports unary and streaming output calls on the service
*/
public static HlcServiceBlockingStub newBlockingStub(
io.grpc.Channel channel) {
io.grpc.stub.AbstractStub.StubFactory<HlcServiceBlockingStub> factory =
new io.grpc.stub.AbstractStub.StubFactory<HlcServiceBlockingStub>() {
@java.lang.Override
public HlcServiceBlockingStub newStub(io.grpc.Channel channel, io.grpc.CallOptions callOptions) {
return new HlcServiceBlockingStub(channel, callOptions);
}
};
return HlcServiceBlockingStub.newStub(factory, channel);
}
/**
* Creates a new ListenableFuture-style stub that supports unary calls on the service
*/
public static HlcServiceFutureStub newFutureStub(
io.grpc.Channel channel) {
io.grpc.stub.AbstractStub.StubFactory<HlcServiceFutureStub> factory =
new io.grpc.stub.AbstractStub.StubFactory<HlcServiceFutureStub>() {
@java.lang.Override
public HlcServiceFutureStub newStub(io.grpc.Channel channel, io.grpc.CallOptions callOptions) {
return new HlcServiceFutureStub(channel, callOptions);
}
};
return HlcServiceFutureStub.newStub(factory, channel);
}
/**
*/
public static abstract class HlcServiceImplBase implements io.grpc.BindableService {
/**
*/
public void touch(cmvr.api.HlcCommand.Touch.Request request,
io.grpc.stub.StreamObserver<cmvr.api.HlcCommand.Touch.Response> responseObserver) {
io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall(getTouchMethod(), responseObserver);
}
@java.lang.Override public final io.grpc.ServerServiceDefinition bindService() {
return io.grpc.ServerServiceDefinition.builder(getServiceDescriptor())
.addMethod(
getTouchMethod(),
io.grpc.stub.ServerCalls.asyncUnaryCall(
new MethodHandlers<
cmvr.api.HlcCommand.Touch.Request,
cmvr.api.HlcCommand.Touch.Response>(
this, METHODID_TOUCH)))
.build();
}
}
/**
*/
public static final class HlcServiceStub extends io.grpc.stub.AbstractAsyncStub<HlcServiceStub> {
private HlcServiceStub(
io.grpc.Channel channel, io.grpc.CallOptions callOptions) {
super(channel, callOptions);
}
@java.lang.Override
protected HlcServiceStub build(
io.grpc.Channel channel, io.grpc.CallOptions callOptions) {
return new HlcServiceStub(channel, callOptions);
}
/**
*/
public void touch(cmvr.api.HlcCommand.Touch.Request request,
io.grpc.stub.StreamObserver<cmvr.api.HlcCommand.Touch.Response> responseObserver) {
io.grpc.stub.ClientCalls.asyncUnaryCall(
getChannel().newCall(getTouchMethod(), getCallOptions()), request, responseObserver);
}
}
/**
*/
public static final class HlcServiceBlockingStub extends io.grpc.stub.AbstractBlockingStub<HlcServiceBlockingStub> {
private HlcServiceBlockingStub(
io.grpc.Channel channel, io.grpc.CallOptions callOptions) {
super(channel, callOptions);
}
@java.lang.Override
protected HlcServiceBlockingStub build(
io.grpc.Channel channel, io.grpc.CallOptions callOptions) {
return new HlcServiceBlockingStub(channel, callOptions);
}
/**
*/
public cmvr.api.HlcCommand.Touch.Response touch(cmvr.api.HlcCommand.Touch.Request request) {
return io.grpc.stub.ClientCalls.blockingUnaryCall(
getChannel(), getTouchMethod(), getCallOptions(), request);
}
}
/**
*/
public static final class HlcServiceFutureStub extends io.grpc.stub.AbstractFutureStub<HlcServiceFutureStub> {
private HlcServiceFutureStub(
io.grpc.Channel channel, io.grpc.CallOptions callOptions) {
super(channel, callOptions);
}
@java.lang.Override
protected HlcServiceFutureStub build(
io.grpc.Channel channel, io.grpc.CallOptions callOptions) {
return new HlcServiceFutureStub(channel, callOptions);
}
/**
*/
public com.google.common.util.concurrent.ListenableFuture<cmvr.api.HlcCommand.Touch.Response> touch(
cmvr.api.HlcCommand.Touch.Request request) {
return io.grpc.stub.ClientCalls.futureUnaryCall(
getChannel().newCall(getTouchMethod(), getCallOptions()), request);
}
}
private static final int METHODID_TOUCH = 0;
private static final class MethodHandlers<Req, Resp> implements
io.grpc.stub.ServerCalls.UnaryMethod<Req, Resp>,
io.grpc.stub.ServerCalls.ServerStreamingMethod<Req, Resp>,
io.grpc.stub.ServerCalls.ClientStreamingMethod<Req, Resp>,
io.grpc.stub.ServerCalls.BidiStreamingMethod<Req, Resp> {
private final HlcServiceImplBase serviceImpl;
private final int methodId;
MethodHandlers(HlcServiceImplBase serviceImpl, int methodId) {
this.serviceImpl = serviceImpl;
this.methodId = methodId;
}
@java.lang.Override
@java.lang.SuppressWarnings("unchecked")
public void invoke(Req request, io.grpc.stub.StreamObserver<Resp> responseObserver) {
switch (methodId) {
case METHODID_TOUCH:
serviceImpl.touch((cmvr.api.HlcCommand.Touch.Request) request,
(io.grpc.stub.StreamObserver<cmvr.api.HlcCommand.Touch.Response>) responseObserver);
break;
default:
throw new AssertionError();
}
}
@java.lang.Override
@java.lang.SuppressWarnings("unchecked")
public io.grpc.stub.StreamObserver<Req> invoke(
io.grpc.stub.StreamObserver<Resp> responseObserver) {
switch (methodId) {
default:
throw new AssertionError();
}
}
}
private static abstract class HlcServiceBaseDescriptorSupplier
implements io.grpc.protobuf.ProtoFileDescriptorSupplier, io.grpc.protobuf.ProtoServiceDescriptorSupplier {
HlcServiceBaseDescriptorSupplier() {}
@java.lang.Override
public com.google.protobuf.Descriptors.FileDescriptor getFileDescriptor() {
return cmvr.api.HlcServiceOuterClass.getDescriptor();
}
@java.lang.Override
public com.google.protobuf.Descriptors.ServiceDescriptor getServiceDescriptor() {
return getFileDescriptor().findServiceByName("HlcService");
}
}
private static final class HlcServiceFileDescriptorSupplier
extends HlcServiceBaseDescriptorSupplier {
HlcServiceFileDescriptorSupplier() {}
}
private static final class HlcServiceMethodDescriptorSupplier
extends HlcServiceBaseDescriptorSupplier
implements io.grpc.protobuf.ProtoMethodDescriptorSupplier {
private final String methodName;
HlcServiceMethodDescriptorSupplier(String methodName) {
this.methodName = methodName;
}
@java.lang.Override
public com.google.protobuf.Descriptors.MethodDescriptor getMethodDescriptor() {
return getServiceDescriptor().findMethodByName(methodName);
}
}
private static volatile io.grpc.ServiceDescriptor serviceDescriptor;
public static io.grpc.ServiceDescriptor getServiceDescriptor() {
io.grpc.ServiceDescriptor result = serviceDescriptor;
if (result == null) {
synchronized (HlcServiceGrpc.class) {
result = serviceDescriptor;
if (result == null) {
serviceDescriptor = result = io.grpc.ServiceDescriptor.newBuilder(SERVICE_NAME)
.setSchemaDescriptor(new HlcServiceFileDescriptorSupplier())
.addMethod(getTouchMethod())
.build();
}
}
}
return result;
}
}

40
cmvr-iot-api/cmvr-iot-edge/cmvr-iot-grpc-lib/src/main/java/cmvr/api/HlcServiceOuterClass.java Normal file
View File

@ -0,0 +1,40 @@
// Generated by the protocol buffer compiler. DO NOT EDIT!
// source: cmvr/api/hlc_service.proto
package cmvr.api;
public final class HlcServiceOuterClass {
private HlcServiceOuterClass() {}
public static void registerAllExtensions(
com.google.protobuf.ExtensionRegistryLite registry) {
}
public static void registerAllExtensions(
com.google.protobuf.ExtensionRegistry registry) {
registerAllExtensions(
(com.google.protobuf.ExtensionRegistryLite) registry);
}
public static com.google.protobuf.Descriptors.FileDescriptor
getDescriptor() {
return descriptor;
}
private static com.google.protobuf.Descriptors.FileDescriptor
descriptor;
static {
java.lang.String[] descriptorData = {
"\n\032cmvr/api/hlc_service.proto\022\010cmvr.api\032\032" +
"cmvr/api/hlc_command.proto2H\n\nHlcService" +
"\022:\n\005touch\022\027.cmvr.api.Touch.Request\032\030.cmv" +
"r.api.Touch.Responseb\006proto3"
};
descriptor = com.google.protobuf.Descriptors.FileDescriptor
.internalBuildGeneratedFileFrom(descriptorData,
new com.google.protobuf.Descriptors.FileDescriptor[] {
cmvr.api.HlcCommand.getDescriptor(),
});
cmvr.api.HlcCommand.getDescriptor();
}
// @@protoc_insertion_point(outer_class_scope)
}

24
cmvr-iot-api/cmvr-iot-edge/cmvr-iot-grpc-lib/src/main/proto/cmvr/api/hlc_command.proto Normal file
View File

@ -0,0 +1,24 @@
syntax = "proto3";
import "cmvr/api/common.proto";
package cmvr.api;
message Touch{
message Request{
CommandHeader.Request header = 1;
// 触点坐标，unit: ms
int32 u = 2; // 水平坐标
int32 v = 3; // 垂直坐标
// 触控时允许的最大作用力，unit: N
double max_force = 4;
}
message Response{
CommandHeader.Feedback header= 1;
}
}

9
cmvr-iot-api/cmvr-iot-edge/cmvr-iot-grpc-lib/src/main/proto/cmvr/api/hlc_service.proto Normal file
View File

@ -0,0 +1,9 @@
syntax = "proto3";
import "cmvr/api/hlc_command.proto";
package cmvr.api;
service HlcService{
rpc touch(Touch.Request) returns (Touch.Response);
}

49
cmvr-iot-api/cmvr-iot-llm/src/main/java/com/cmvr/llm/service/impl/LLMTouchServiceImpl.java
View File

@ -1,6 +1,8 @@
package com.cmvr.llm.service.impl;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONObject;
@ -8,12 +10,14 @@ import com.cmvr.common.exception.GlobalException;
import com.cmvr.common.utils.http.CallAPIUtil;
import com.cmvr.llm.config.APIProperties;
import com.cmvr.llm.service.LLMTouchService;
import com.cmvr.llm.util.LargeModelFileUploadUtil;
import com.cmvr.system.domain.vo.CarIcon;
import com.cmvr.system.service.CarPathFinderService;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.InputStream;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -95,26 +99,29 @@ public class LLMTouchServiceImpl implements LLMTouchService {
String iconName = nextStepIcon.getIconName();
try {
// 上传截图
// byte[] imageBytes = FileUtil.readBytes(image);
// imageUrl = LargeModelFileUploadUtil.uploadFile(imageBytes);
//
// // 拼接图标路径，例如 icons/honda/sedan/空调控制.png
// String iconPath = StrUtil.format("icon/{}/{}/{}.png", manufacturer, vehType, iconName);
//
// // 从 resources 加载图标文件
// InputStream iconStream = this.getClass().getClassLoader().getResourceAsStream(iconPath);
// if (iconStream == null) {
// throw new GlobalException("未找到图标资源文件：" + iconPath);
// }
//
// // 上传图标
// byte[] iconBytes = IoUtil.readBytes(iconStream);
// iconImageUrl = LargeModelFileUploadUtil.uploadFile(iconBytes);
imageUrl = "https://aiagentplatform.cmft.com/api/proxy/down?Action=Download&Version=2022-01-01&Path=upload/full/5d/a3/07244360c69636d5c146e5470d8d5ddf12ceed2643ffd892b7b7a5c4aaeb&IsAnonymous=true";
iconImageUrl =
iconName.equals("驾驶偏好") ?
"https://aiagentplatform.cmft.com/api/proxy/down?Action=Download&Version=2022-01-01&Path=upload/full/a4/1b/6a7319709844c0f483a1d7adcfdbbff478ff5e34bed9ecff5d8fd407f3ac&IsAnonymous=true"
: "https://aiagentplatform.cmft.com/api/proxy/down?Action=Download&Version=2022-01-01&Path=upload%2Ffull%2Ff1%2Fb9%2F8fb5f96f0299ccef60511eb8f7cfdcec890e563bc8e00854e63484086953&IsAnonymous=true";
byte[] imageBytes = FileUtil.readBytes(image);
imageUrl = LargeModelFileUploadUtil.uploadFile(imageBytes);
// 拼接图标路径，例如 icons/honda/sedan/空调控制.png
String iconPath = StrUtil.format("icon/{}/{}/{}.png", manufacturer, vehType, iconName);
// 从 resources 加载图标文件
InputStream iconStream = this.getClass().getClassLoader().getResourceAsStream(iconPath);
if (iconStream == null) {
throw new GlobalException("未找到图标资源文件：" + iconPath);
}
// 上传图标
byte[] iconBytes = IoUtil.readBytes(iconStream);
// 设置的icon
// iconImageUrl = "https://aiagentplatform.cmft.com/api/proxy/down?Action=Download&Version=2022-01-01&Path=upload%2Ffull%2F8f%2F1a%2F1b0b791926b89d1c748c9cb1280bb3e4697543b3a715e692565107442c9c&IsAnonymous=true";
iconImageUrl = LargeModelFileUploadUtil.uploadFile(iconBytes);
// imageUrl = "https://aiagentplatform.cmft.com/api/proxy/down?Action=Download&Version=2022-01-01&Path=upload%2Ffull%2F88%2F99%2Fe4452191c9706d107c17b27dc0a08e13a3a1a85a261eec802ae85c11f26c&IsAnonymous=true";
// imageUrl = "https://aiagentplatform.cmft.com/api/proxy/down?Action=Download&Version=2022-01-01&Path=upload/full/5d/a3/07244360c69636d5c146e5470d8d5ddf12ceed2643ffd892b7b7a5c4aaeb&IsAnonymous=true";
// iconImageUrl =
// iconName.equals("驾驶偏好") ?
// "https://aiagentplatform.cmft.com/api/proxy/down?Action=Download&Version=2022-01-01&Path=upload/full/a4/1b/6a7319709844c0f483a1d7adcfdbbff478ff5e34bed9ecff5d8fd407f3ac&IsAnonymous=true"
// : "https://aiagentplatform.cmft.com/api/proxy/down?Action=Download&Version=2022-01-01&Path=upload%2Ffull%2Ff1%2Fb9%2F8fb5f96f0299ccef60511eb8f7cfdcec890e563bc8e00854e63484086953&IsAnonymous=true";
} catch (Exception e) {
throw new GlobalException("上传图标或截图失败：" + e.getMessage());
}
@ -128,7 +135,7 @@ public class LLMTouchServiceImpl implements LLMTouchService {
}
// 7) 返回
result.put("isFinish", false);
result.put("isFinish", path.size() == 1);
result.put("coordinates", coordinates);
return result;

8
cmvr-iot-system/src/main/java/com/cmvr/system/util/CarPathFinderUtil.java
View File

@ -1,5 +1,6 @@
package com.cmvr.system.util;
import cn.hutool.core.util.ObjUtil;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.cmvr.system.domain.vo.CarIcon;
@ -52,7 +53,7 @@ public class CarPathFinderUtil {
}
}
uiMap.put(ui.getUiName(), ui);
uiMap.put(ui.getUiImage(), ui);
}
return uiMap;
@ -83,7 +84,10 @@ public class CarPathFinderUtil {
}
for (CarIcon icon : currentUi.getIcons()) {
if (icon.getIconName().equals(targetFeature)) {
if (ObjUtil.isEmpty(icon.getIconImage())) {
continue;
}
if (icon.getIconImage().equals(targetFeature)) {
currentPath.add(icon);
return currentPath;
}

9
cmvr-iot-test/src/main/java/com/cmvr/test/flow/runtime/engine/FlowTaskEngine.java
View File

@ -51,14 +51,6 @@ public class FlowTaskEngine {
context.setRunParams(jsonObject);
}
flowTaskExecutor.executeGraph(message, item, latch::countDown);
// flowTaskExecutor.executeGraph(message, item, () -> {
// try {
// // 可以放一些 per-item 回调逻辑
// } finally {
// latch.countDown();
// }
// });
try {
latch.await(); // 等待该检测项的子流程全部完成
} catch (InterruptedException e) {
@ -66,7 +58,6 @@ public class FlowTaskEngine {
log.error("检测项执行被中断：itemId={}", itemId, e);
break;
}
// flowTaskExecutor.executeGraph(message, item, onFinished);
}
}
}

8
cmvr-iot-test/src/main/java/com/cmvr/test/flow/runtime/engine/FlowTaskRuntimeEntry.java
View File

@ -64,7 +64,7 @@ public class FlowTaskRuntimeEntry implements FlowTaskRuntimeService {
try {
// 注册上下文
registerTaskContext(instId, taskId, terminalId, null, RunModeEnum.NORMAL, details, taskExecuteNormalVO.getRunParams());
registerTaskContext(instId, taskId, terminalId, null, RunModeEnum.NORMAL, taskExecuteNormalVO.getRunParams());
// 异步提交执行
flowTaskAsyncDispatcher.submit(instId, taskId, terminalId, RunModeEnum.NORMAL, details);
@ -115,7 +115,7 @@ public class FlowTaskRuntimeEntry implements FlowTaskRuntimeService {
List<TeQueryTaskDetailDTO> details = CollUtil.newArrayList(teQueryTaskDetailDTO);
// 注册上下文
registerTaskContext(instId, taskId, terminalId, itemId, RunModeEnum.TRIAL, details, runParams);
registerTaskContext(instId, taskId, terminalId, itemId, RunModeEnum.TRIAL, runParams);
// 异步调度执行（与正式任务一致，只是模式为 TRIAL）
flowTaskAsyncDispatcher.submit(instId, taskId, terminalId, RunModeEnum.TRIAL, details);
@ -139,7 +139,6 @@ public class FlowTaskRuntimeEntry implements FlowTaskRuntimeService {
private TeTaskInst createAndSaveTaskInstance(String taskId, String terminalId, RunModeEnum runMode) {
TeTaskInst instance = new TeTaskInst();
instance.setTaskId(taskId);
// instance.setConfig(JSON.toJSONString(taskExecuteVO.getConfig()));
instance.setTerminalId(terminalId);
instance.setRunMode(runMode.name());
instance.setStatus(String.valueOf(TaskStatusEnum.RUNNING.getCode()));
@ -154,8 +153,7 @@ public class FlowTaskRuntimeEntry implements FlowTaskRuntimeService {
* 注册上下文
*/
private void registerTaskContext(String instId, String taskId, String terminalId,
String itemId, RunModeEnum mode, List<TeQueryTaskDetailDTO> details,
JSONObject runParams) {
String itemId, RunModeEnum mode, JSONObject runParams) {
TaskContext ctx = new TaskContext();
ctx.setInstId(instId);
ctx.setTaskId(taskId);

19
cmvr-iot-test/src/main/java/com/cmvr/test/flow/runtime/operator/edge/EdgeCameraOperateService.java
View File

@ -55,16 +55,17 @@ public class EdgeCameraOperateService implements EdgeOperateService {
}
case CAMERA_GETRGBIMAGE: {
String imageUrl;
// String imageUrl;
// String imageUrl = StrUtil.format("id为 [{}] 的相机拍照了", deviceId);
if (message.getLoopNum() <= 1) {
// 车辆控制调节
imageUrl = "http://192.168.1.100:9000/cmvr-iot/FILE/20250821/1755761816340.jpg";
} else {
// 驾驶偏好
imageUrl = "http://192.168.1.100:9000/cmvr-iot/FILE/20250822/1755849654731.jpg";
}
// String imageUrl = edgeCameraService.getRGBImage(terminalId, deviceId);
// if (message.getLoopNum() <= 1) {
// // 车辆控制调节
// imageUrl = "http://192.168.1.100:9000/cmvr-iot/FILE/20250821/1755761816340.jpg";
// } else {
// // 驾驶偏好
// imageUrl = "http://192.168.1.100:9000/cmvr-iot/FILE/20250822/1755849654731.jpg";
// }
// String imageUrl = "http://192.168.1.100:9000/cmvr-iot/IMAGE/20250901/1756718594725.jpg";
String imageUrl = edgeCameraService.getRGBDImages(terminalId, deviceId);
JSONArray imageUrls = new JSONArray();
if (ObjUtil.isNotEmpty(upstreamOutput)) {

5
cmvr-iot-test/src/main/java/com/cmvr/test/flow/runtime/operator/llm/LLMTouchOperateService.java
View File

@ -6,6 +6,7 @@ import com.alibaba.fastjson2.JSONObject;
import com.cmvr.common.config.properties.MinioProperties;
import com.cmvr.common.core.minio.MinioService;
import com.cmvr.common.exception.GlobalException;
import com.cmvr.edge.client.service.EdgeHlcService;
import com.cmvr.llm.service.LLMTouchService;
import com.cmvr.test.enums.ActionEnum;
import com.cmvr.test.flow.builder.FlowNodeWrapper;
@ -27,6 +28,7 @@ public class LLMTouchOperateService implements LLMOperateService {
private final MinioService minioService;
private final MinioProperties minioProps;
private final LLMTouchService llmTouchService;
private final EdgeHlcService edgeHlcService;
@Override
public boolean supports(ActionEnum action) {
@ -38,6 +40,8 @@ public class LLMTouchOperateService implements LLMOperateService {
ActionEnum action = message.getAction();
String instId = message.getInstId();
JSONObject inputParams = message.getInputParams();
String deviceId = inputParams.getString("deviceId");
String terminalId = message.getTerminalId();
JSONObject output = new JSONObject();
switch (action) {
@ -76,6 +80,7 @@ public class LLMTouchOperateService implements LLMOperateService {
output.put("coordinates", cal);
// todo 获取坐标后调用边缘系统
edgeHlcService.touch(image, terminalId, "hlc01", x, y, 100d);
} catch (Exception e) {
throw new RuntimeException(e);
}

28
cmvr-iot-vi/pom.xml Normal file
View File

@ -0,0 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.cmvr</groupId>
<artifactId>cmvr-iot</artifactId>
<version>3.8.9</version>
</parent>
<artifactId>cmvr-iot-vi</artifactId>
<description>
语音交互模块
</description>
<dependencies>
<!-- 通用工具-->
<dependency>
<groupId>com.cmvr</groupId>
<artifactId>cmvr-iot-common</artifactId>
</dependency>
</dependencies>
</project>

12
cmvr-iot-vi/src/main/java/com/cmvr/vi/mapper/ViCorpusMapper.java Normal file
View File

@ -0,0 +1,12 @@
package com.cmvr.vi.mapper;
import com.cmvr.vi.model.domain.ViCorpus;
import com.github.yulichang.base.MPJBaseMapper;
/**
* 语料库Mapper接口
*
* @author cmvr-iot
*/
public interface ViCorpusMapper extends MPJBaseMapper<ViCorpus> {
}

91
cmvr-iot-vi/src/main/java/com/cmvr/vi/model/domain/ViCorpus.java Normal file
View File

@ -0,0 +1,91 @@
package com.cmvr.vi.model.domain;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.cmvr.common.annotation.Excel;
import com.cmvr.common.core.domain.BaseEntity;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* 语料库对象 vi_corpus
*
* @author cmvr-iot
*/
@Data
@ApiModel("语料库对象")
public class ViCorpus extends BaseEntity {
private static final long serialVersionUID = 1L;
/**
* 语料ID
*/
@ApiModelProperty("语料ID")
@TableId(type = IdType.AUTO)
private Long corpusId;
/**
* 语料名称
*/
@Excel(name = "语料名称")
@ApiModelProperty("语料名称")
private String corpusName;
/**
* 语料类型（WAKEUP唤醒 / TEST测试）
*/
@Excel(name = "语料类型", readConverterExp = "WAKEUP唤醒,TEST测试")
@ApiModelProperty("语料类型")
private String type;
/**
* 语音对应的文本
*/
@Excel(name = "语音对应的文本")
@ApiModelProperty("语音对应的文本")
private String textContent;
/**
* 语音文件存放路径
*/
@Excel(name = "语音文件存放路径")
@ApiModelProperty("语音文件存放路径")
private String audioPath;
/**
* 音色
*/
@Excel(name = "音色")
@ApiModelProperty("音色")
private String voiceType;
/**
* 方言
*/
@Excel(name = "方言")
@ApiModelProperty("方言")
private String dialect;
/**
* 预期结果（预留）
*/
// @Excel(name = "预期结果(预留)")
@ApiModelProperty("预期结果(预留)")
private String expectedResult;
/**
* 同一父语料下的顺序
*/
@Excel(name = "同一父语料下的顺序")
@ApiModelProperty("同一父语料下的顺序")
private Integer sortOrder;
/**
* 状态（0正常 1停用）
*/
@Excel(name = "状态", readConverterExp = "0=正常,1=停用")
@ApiModelProperty(value = "状态", notes = "0=正常,1=停用")
private String status;
}

54
cmvr-iot-vi/src/main/java/com/cmvr/vi/service/IViCorpusService.java Normal file
View File

@ -0,0 +1,54 @@
package com.cmvr.vi.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.cmvr.vi.model.domain.ViCorpus;
import java.util.List;
/**
* 语料库Service接口
*
* @author cmvr-iot
*/
public interface IViCorpusService extends IService<ViCorpus> {
/**
* 查询语料库
*
* @param corpusId 语料库主键
* @return 语料库
*/
public ViCorpus selectViCorpusByCorpusId(Long corpusId);
/**
* 查询语料库列表
*
* @param viCorpus 语料库
* @return 语料库集合
*/
public List<ViCorpus> selectViCorpusList(ViCorpus viCorpus);
/**
* 新增语料库
*
* @param viCorpus 语料库
* @return 结果
*/
public int insertViCorpus(ViCorpus viCorpus);
/**
* 修改语料库
*
* @param viCorpus 语料库
* @return 结果
*/
public int updateViCorpus(ViCorpus viCorpus);
/**
* 批量删除语料库
*
* @param corpusIds 需要删除的语料库主键集合
* @return 结果
*/
public int deleteViCorpusByCorpusIds(Long[] corpusIds);
}

79
cmvr-iot-vi/src/main/java/com/cmvr/vi/service/impl/IViCorpusServiceImpl.java Normal file
View File

@ -0,0 +1,79 @@
package com.cmvr.vi.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.cmvr.vi.mapper.ViCorpusMapper;
import com.cmvr.vi.model.domain.ViCorpus;
import com.cmvr.vi.service.IViCorpusService;
import org.springframework.stereotype.Service;
import java.util.Arrays;
import java.util.List;
/**
* 语料库Service业务层处理
*
* @author cmvr-iot
*/
@Service
public class IViCorpusServiceImpl extends ServiceImpl<ViCorpusMapper, ViCorpus> implements IViCorpusService {
/**
* 查询语料库
*
* @param corpusId 语料库主键
* @return 语料库
*/
@Override
public ViCorpus selectViCorpusByCorpusId(Long corpusId) {
return this.getById(corpusId);
}
/**
* 查询语料库列表
*
* @param viCorpus 语料库
* @return 语料库
*/
@Override
public List<ViCorpus> selectViCorpusList(ViCorpus viCorpus) {
LambdaQueryWrapper<ViCorpus> wrapper = Wrappers.lambdaQuery();
wrapper.eq(ViCorpus::getType, viCorpus.getType());
return this.list(wrapper);
}
/**
* 新增语料库
*
* @param viCorpus 语料库
* @return 结果
*/
@Override
public int insertViCorpus(ViCorpus viCorpus) {
return this.baseMapper.insert(viCorpus);
}
/**
* 修改语料库
*
* @param viCorpus 语料库
* @return 结果
*/
@Override
public int updateViCorpus(ViCorpus viCorpus) {
return this.baseMapper.updateById(viCorpus);
}
/**
* 批量删除语料库
*
* @param corpusIds 需要删除的语料库主键
* @return 结果
*/
@Override
public int deleteViCorpusByCorpusIds(Long[] corpusIds) {
return this.baseMapper.deleteBatchIds(Arrays.asList(corpusIds));
}
}

8
pom.xml
View File

@ -260,6 +260,13 @@
<version>${cmvr-iot.version}</version>
</dependency>
<!-- 语音交互-->
<dependency>
<groupId>com.cmvr</groupId>
<artifactId>cmvr-iot-vi</artifactId>
<version>${cmvr-iot.version}</version>
</dependency>
<!-- 大模型调用-->
<dependency>
<groupId>com.cmvr</groupId>
@ -314,6 +321,7 @@
<module>cmvr-iot-device</module>
<module>cmvr-iot-test</module>
<module>cmvr-iot-api</module>
<module>cmvr-iot-vi</module>
</modules>
<packaging>pom</packaging>