fix: 触控坐标计算

cmvr-iot-api/cmvr-iot-edge/cmvr-iot-grpc-client/src/main/java/com/cmvr/edge/client/service/impl/EdgeCameraServiceImpl.java

@@ -159,7 +159,31 @@ public class EdgeCameraServiceImpl implements EdgeCameraService, EdgeStreamServi
                 .build();
         CameraCommand.GetRGBDImagesCommand.Feedback feedback = executeGrpcCall(() -> stub.getRGBDImages(request));
 
-        return JSON.toJSONString(feedback);
+        // 彩色图和深度图
+        CameraCommand.FrameData frame = feedback.getColorFrame();
+        CameraCommand.FrameData depthFrame = feedback.getDepthFrame();
+        if (!feedback.hasColorFrame() || frame.getWidth() <= 0 || frame.getHeight() <= 0) {
+            throw new GlobalException("响应图片数据无效");
+        }
+        String filePath = saveImage(frame.getData(), frame.getWidth(), frame.getHeight(), false);
+        String deepFilePath = saveImage(depthFrame.getData(), depthFrame.getWidth(), depthFrame.getHeight(), true);
+        List<String> result = new ArrayList<>();
+        try {
+            MultipartFile file = convertFileToMultipartFile(filePath);
+            MultipartFile deepFile = convertFileToMultipartFile(deepFilePath);
+            String imageUrl = sysFileService.uploadFile(file, FileType.IMAGE.code());
+            result.add(imageUrl);
+            String deepImageUrl = sysFileService.uploadFile(deepFile, FileType.IMAGE.code());
+            result.add(deepImageUrl);
+        } catch (IOException e) {
+            throw new GlobalException("图片上传失败: " + e.getMessage());
+        } finally {
+            deleteTempFile(filePath);
+            deleteTempFile(deepFilePath);
+//             todo 暂时不关闭摄像头
+//            stop(terminalId, deviceId);
+        }
+        return JSON.toJSONString(result);
     }
 
     // 缓冲区和收集状态
@@ -338,6 +362,56 @@ public class EdgeCameraServiceImpl implements EdgeCameraService, EdgeStreamServi
         return filePath;
     }
 
+    private String saveImage(ByteString pixelData, int width, int height, boolean isDepth) {
+        File tempDir = new File(System.getProperty("java.io.tmpdir"));
+        FileUtil.mkdir(tempDir);
+
+        String fileName = System.currentTimeMillis() + (isDepth ? "_depth" : "") + ".jpg";
+        String filePath = Paths.get(tempDir.getAbsolutePath(), fileName).toString();
+        try {
+            byte[] pixels = pixelData.toByteArray();
+            BufferedImage image;
+
+            if (isDepth) {
+                // 深度图：单通道（灰度）
+                DataBuffer buffer = new DataBufferByte(pixels, pixels.length);
+                WritableRaster raster = Raster.createInterleavedRaster(
+                        buffer, width, height, width, 1,
+                        new int[]{0}, null
+                );
+
+                ColorModel cm = new ComponentColorModel(
+                        ColorSpace.getInstance(ColorSpace.CS_GRAY),
+                        false, false,
+                        Transparency.OPAQUE,
+                        DataBuffer.TYPE_BYTE
+                );
+                image = new BufferedImage(cm, raster, false, null);
+            } else {
+                // 彩色图：RGB 三通道
+                DataBuffer buffer = new DataBufferByte(pixels, pixels.length);
+                WritableRaster raster = Raster.createInterleavedRaster(
+                        buffer, width, height, 3 * width, 3,
+                        new int[]{2, 1, 0}, null
+                );
+
+                ColorModel cm = new ComponentColorModel(
+                        ColorSpace.getInstance(ColorSpace.CS_sRGB),
+                        false, false,
+                        Transparency.OPAQUE,
+                        DataBuffer.TYPE_BYTE
+                );
+                image = new BufferedImage(cm, raster, false, null);
+            }
+
+            File output = new File(filePath);
+            ImageIO.write(image, "jpg", output);
+        } catch (IOException e) {
+            throw new GlobalException(e.getMessage());
+        }
+        return filePath;
+    }
+
     /**
      * 将本地文件转换为 MultipartFile
      */

cmvr-iot-api/cmvr-iot-edge/cmvr-iot-grpc-lib/src/main/proto/cmvr/api/camera_command.proto

@@ -21,6 +21,14 @@ message FrameData {
     bool is_key_frame = 6;
 }
 
+message Rs2Intrinsics {
+  float cx = 1;                           // 主点水平坐标（从左边缘的像素偏移）
+  float cy = 2;                           // 主点垂直坐标（从上边缘的像素偏移）
+  float fx = 3;                            // x方向焦距（像素宽度的倍数）
+  float fy = 4;                            // y方向焦距（像素高度的倍数）
+  repeated float coeffs = 5 [packed = true]; // 畸变系数（固定5个元素）
+}
+
 message CameraState {
     bool is_initialized = 1;
     bool is_opened = 2;
@@ -71,6 +79,7 @@ message GetRGBImageCommand {
     message Feedback {
         CommandHeader.Feedback header = 1;
         FrameData color_frame = 2;
+        Rs2Intrinsics intrinsics = 3;
     }
 }
 
@@ -82,6 +91,7 @@ message GetDepthImageCommand {
     message Feedback {
         CommandHeader.Feedback header = 1;
         FrameData depth_frame = 2;
+        Rs2Intrinsics intrinsics = 3;
     }
 }
 
@@ -94,6 +104,7 @@ message GetRGBDImagesCommand {
         CommandHeader.Feedback header = 1;
         FrameData color_frame = 2;
         FrameData depth_frame = 3;
+        Rs2Intrinsics intrinsics = 4;
     }
 }
 
@@ -126,7 +137,8 @@ message GetRGBImageStreamCommand {
     message Feedback {
         CommandHeader.Feedback header = 1;
         FrameData color_frame = 2;
-        int32 seq_no = 3;
+        Rs2Intrinsics intrinsics = 3;
+        int32 seq_no = 4;
     }
 }
 
@@ -139,7 +151,8 @@ message GetDepthImageStreamCommand {
     message Feedback {
         CommandHeader.Feedback header = 1;
         FrameData depth_frame = 2;
-        int32 seq_no = 3;
+        Rs2Intrinsics intrinsics = 3;
+        int32 seq_no = 4;
     }
 }
 
@@ -153,7 +166,8 @@ message GetRGBDImagesStreamCommand {
         CommandHeader.Feedback header = 1;
         FrameData color_frame = 2;
         FrameData depth_frame = 3;
-        int32 seq_no = 4;
+        Rs2Intrinsics intrinsics = 4;
+        int32 seq_no = 5;
     }
 }
 

cmvr-iot-test/src/main/java/com/cmvr/test/flow/runtime/operator/llm/LLMTouchOperateService.java

@@ -62,6 +62,20 @@ public class LLMTouchOperateService implements LLMOperateService {
                 try {
                     File image = minioService.getFile(bucketName, objectName);
                     output = llmTouchService.touchCoordinates(image, targetFeature, manufacturer, vehType);
+
+                    // 计算坐标
+                    JSONArray coords = output.getJSONArray("coordinates"); // [x1, y1, x2, y2]
+                    int x1 = coords.getIntValue(0);
+                    int y1 = coords.getIntValue(1);
+                    int x2 = coords.getIntValue(2);
+                    int y2 = coords.getIntValue(3);
+
+                    int x = (x1 + x2) / 2;
+                    int y = (y1 + y2) / 2;
+                    String cal = StrUtil.format("{},{}", x, y);
+                    output.put("coordinates", cal);
+
+                    // todo 获取坐标后调用边缘系统
                 } catch (Exception e) {
                     throw new RuntimeException(e);
                 }
@@ -71,6 +85,7 @@ public class LLMTouchOperateService implements LLMOperateService {
             default:
                 throw new GlobalException("不支持的触控操作类型: " + action);
         }
+
         return TaskNodeExecuteResult.success(output);
     }
 }