SmartBench/CMVR-IOT
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 触控坐标计算

Browse Source
This commit is contained in:
stream 2025-08-28 16:31:06 +08:00
parent 8e1b75b7f9
commit 91eeb0d586
4 changed files with 2418 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
cmvr-iot-api/cmvr-iot-edge/cmvr-iot-grpc-client/src/main/java/com/cmvr/edge/client/service/impl/EdgeCameraServiceImpl.java
View File

@ -159,7 +159,31 @@ public class EdgeCameraServiceImpl implements EdgeCameraService, EdgeStreamServi
.build();
CameraCommand.GetRGBDImagesCommand.Feedback feedback = executeGrpcCall(() -> stub.getRGBDImages(request));
return JSON.toJSONString(feedback);
// 彩色图和深度图
CameraCommand.FrameData frame = feedback.getColorFrame();
CameraCommand.FrameData depthFrame = feedback.getDepthFrame();
if (!feedback.hasColorFrame() || frame.getWidth() <= 0 || frame.getHeight() <= 0) {
throw new GlobalException("响应图片数据无效");
}
String filePath = saveImage(frame.getData(), frame.getWidth(), frame.getHeight(), false);
String deepFilePath = saveImage(depthFrame.getData(), depthFrame.getWidth(), depthFrame.getHeight(), true);
List<String> result = new ArrayList<>();
try {
MultipartFile file = convertFileToMultipartFile(filePath);
MultipartFile deepFile = convertFileToMultipartFile(deepFilePath);
String imageUrl = sysFileService.uploadFile(file, FileType.IMAGE.code());
result.add(imageUrl);
String deepImageUrl = sysFileService.uploadFile(deepFile, FileType.IMAGE.code());
result.add(deepImageUrl);
} catch (IOException e) {
throw new GlobalException("图片上传失败: " + e.getMessage());
} finally {
deleteTempFile(filePath);
deleteTempFile(deepFilePath);
// todo 暂时不关闭摄像头
// stop(terminalId, deviceId);
}
return JSON.toJSONString(result);
}
// 缓冲区和收集状态
@ -338,6 +362,56 @@ public class EdgeCameraServiceImpl implements EdgeCameraService, EdgeStreamServi
return filePath;
}
private String saveImage(ByteString pixelData, int width, int height, boolean isDepth) {
File tempDir = new File(System.getProperty("java.io.tmpdir"));
FileUtil.mkdir(tempDir);
String fileName = System.currentTimeMillis() + (isDepth ? "_depth" : "") + ".jpg";
String filePath = Paths.get(tempDir.getAbsolutePath(), fileName).toString();
try {
byte[] pixels = pixelData.toByteArray();
BufferedImage image;
if (isDepth) {
// 深度图：单通道（灰度）
DataBuffer buffer = new DataBufferByte(pixels, pixels.length);
WritableRaster raster = Raster.createInterleavedRaster(
buffer, width, height, width, 1,
new int[]{0}, null
);
ColorModel cm = new ComponentColorModel(
ColorSpace.getInstance(ColorSpace.CS_GRAY),
false, false,
Transparency.OPAQUE,
DataBuffer.TYPE_BYTE
);
image = new BufferedImage(cm, raster, false, null);
} else {
// 彩色图：RGB 三通道
DataBuffer buffer = new DataBufferByte(pixels, pixels.length);
WritableRaster raster = Raster.createInterleavedRaster(
buffer, width, height, 3 * width, 3,
new int[]{2, 1, 0}, null
);
ColorModel cm = new ComponentColorModel(
ColorSpace.getInstance(ColorSpace.CS_sRGB),
false, false,
Transparency.OPAQUE,
DataBuffer.TYPE_BYTE
);
image = new BufferedImage(cm, raster, false, null);
}
File output = new File(filePath);
ImageIO.write(image, "jpg", output);
} catch (IOException e) {
throw new GlobalException(e.getMessage());
}
return filePath;
}
/**
* 将本地文件转换为 MultipartFile
*/

2416
cmvr-iot-api/cmvr-iot-edge/cmvr-iot-grpc-lib/src/main/java/cmvr/api/CameraCommand.java
View File

File diff suppressed because it is too large Load Diff

20
cmvr-iot-api/cmvr-iot-edge/cmvr-iot-grpc-lib/src/main/proto/cmvr/api/camera_command.proto
View File

@ -21,6 +21,14 @@ message FrameData {
bool is_key_frame = 6;
}
message Rs2Intrinsics {
float cx = 1; // 主点水平坐标（从左边缘的像素偏移）
float cy = 2; // 主点垂直坐标（从上边缘的像素偏移）
float fx = 3; // x方向焦距（像素宽度的倍数）
float fy = 4; // y方向焦距（像素高度的倍数）
repeated float coeffs = 5 [packed = true]; // 畸变系数（固定5个元素）
}
message CameraState {
bool is_initialized = 1;
bool is_opened = 2;
@ -71,6 +79,7 @@ message GetRGBImageCommand {
message Feedback {
CommandHeader.Feedback header = 1;
FrameData color_frame = 2;
Rs2Intrinsics intrinsics = 3;
}
}
@ -82,6 +91,7 @@ message GetDepthImageCommand {
message Feedback {
CommandHeader.Feedback header = 1;
FrameData depth_frame = 2;
Rs2Intrinsics intrinsics = 3;
}
}
@ -94,6 +104,7 @@ message GetRGBDImagesCommand {
CommandHeader.Feedback header = 1;
FrameData color_frame = 2;
FrameData depth_frame = 3;
Rs2Intrinsics intrinsics = 4;
}
}
@ -126,7 +137,8 @@ message GetRGBImageStreamCommand {
message Feedback {
CommandHeader.Feedback header = 1;
FrameData color_frame = 2;
int32 seq_no = 3;
Rs2Intrinsics intrinsics = 3;
int32 seq_no = 4;
}
}
@ -139,7 +151,8 @@ message GetDepthImageStreamCommand {
message Feedback {
CommandHeader.Feedback header = 1;
FrameData depth_frame = 2;
int32 seq_no = 3;
Rs2Intrinsics intrinsics = 3;
int32 seq_no = 4;
}
}
@ -153,7 +166,8 @@ message GetRGBDImagesStreamCommand {
CommandHeader.Feedback header = 1;
FrameData color_frame = 2;
FrameData depth_frame = 3;
int32 seq_no = 4;
Rs2Intrinsics intrinsics = 4;
int32 seq_no = 5;
}
}

15
cmvr-iot-test/src/main/java/com/cmvr/test/flow/runtime/operator/llm/LLMTouchOperateService.java
View File

@ -62,6 +62,20 @@ public class LLMTouchOperateService implements LLMOperateService {
try {
File image = minioService.getFile(bucketName, objectName);
output = llmTouchService.touchCoordinates(image, targetFeature, manufacturer, vehType);
// 计算坐标
JSONArray coords = output.getJSONArray("coordinates"); // [x1, y1, x2, y2]
int x1 = coords.getIntValue(0);
int y1 = coords.getIntValue(1);
int x2 = coords.getIntValue(2);
int y2 = coords.getIntValue(3);
int x = (x1 + x2) / 2;
int y = (y1 + y2) / 2;
String cal = StrUtil.format("{},{}", x, y);
output.put("coordinates", cal);
// todo 获取坐标后调用边缘系统
} catch (Exception e) {
throw new RuntimeException(e);
}
@ -71,6 +85,7 @@ public class LLMTouchOperateService implements LLMOperateService {
default:
throw new GlobalException("不支持的触控操作类型: " + action);
}
return TaskNodeExecuteResult.success(output);
}
}