diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e69de29 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7bcf26b --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.venv/ +.pytest_cache/ +__pycache__/ +*.py[cod] +*.egg-info/ diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..c8cfe39 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.10 diff --git a/README.md b/README.md index 43d148d..5f8d801 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,14 @@ `event_id` 作为幂等键； - `RobotCommand -> ApprovedRobotCommand` 安全门和无重试的类型化 AGV 命令映射； - 文本/JSON 日志、共享 gRPC Channel 与 HTTP Client； -- 可直接执行的 smoke、PPE 检测和对话占位配置。 +- 可直接执行的最小测试 fixture，以及包含 PPE 检测和对话占位链路的统一部署配置。 -`detect_server/pipeline.yaml` 当前只启用 Construction PPE 模型，并经过时间窗口规则向 -8081 上报告警。六类 PPE 模型及第二 HTTP 平台的实现仍保留在注册表和连接器中，但不在 -当前 Pipeline 图中实例化，因此不会加载第二份权重、执行第二次推理或访问 8082。 -VAD/ASR/LLM/TTS 尚未内置；`talk_server/pipeline.yaml` 仍使用模拟音频数据，等待 -cmvr-es 音频双向流 proto 落地。 +`configs/edge_ai.yaml` 是统一部署配置，其中同时定义 `detection` 和 `talk` 两个 +Pipeline。`detection` 当前只启用 Construction PPE 模型，并经过时间窗口规则向 8081 +上报告警。六类 PPE 模型及第二 HTTP 平台的实现仍保留在注册表和连接器中，但不在当前 +Pipeline 图中实例化，因此不会加载第二份权重、执行第二次推理或访问 8082。 +VAD/ASR/LLM/TTS 尚未内置；`talk` 仍使用模拟音频数据，等待 cmvr-es 音频双向流 +proto 落地。 ## 架构概览 @@ -46,12 +47,18 @@ cmvr_edge_ai/ ├── .python-version # uv 默认 Python 3.10 ├── uv.lock # 所有 profile 的可复现依赖锁 ├── configs/ -│ └── smoke.yaml # 不依赖外部服务的最小运行验证 +│ ├── edge_ai.yaml # detection + talk 统一部署配置 +│ └── debug/ +│ └── detection_viewer.yaml # 远端相机 -> PPE 检测 -> 本地画框窗口 ├── detect_server/ -│ └── pipeline.yaml # cmvr-es 相机 -> PPE 告警 -> HTTP 平台 +│ ├── README.md # PPE 检测链路与 Viewer 使用说明 +│ └── show_detections.py # OpenCV 实时检测结果 Demo +├── models/ +│ └── detection/ # 按模型 ID/版本组织的检测模型制品库 +│ ├── construction-ppe-yolov8/v1/ # best.pt + 独立 model card +│ └── ppe-6classes-yolov8n/v1/ # best.pt + 独立 model card ├── talk_server/ -│ ├── nodes/ # 对话插件预留目录 -│ └── pipeline.yaml # 模拟音频 -> 对话占位 -> 日志 +│ └── nodes/ # 对话插件预留目录 ├── scripts/ │ ├── bootstrap.sh # 一键创建 uv 环境、生成 bindings 并自检 │ └── generate_cmvr_stubs.py # 从 cmvr-es proto 生成 Python bindings @@ -69,6 +76,8 @@ cmvr_edge_ai/ │ ├── compiler.py # 配置到可执行 DAG 的编译器 │ └── cli.py # validate/run/plugins/models └── tests/ + └── fixtures/ + └── minimal_pipeline.yaml # 不依赖外部服务的框架/CLI 自检配置 ``` ## 快速开始 @@ -98,11 +107,17 @@ bash scripts/bootstrap.sh --profile core 机器上重新选择依赖版本。无需 `source .venv/bin/activate`，统一通过 `uv run --no-sync` 使用已经安装好的环境： +配置中的相对文件路径按进程启动时的当前工作目录（`cwd`）解析，不是按 YAML +文件所在目录解析。因此本文的 bootstrap、validate、run 和 Viewer 命令都应从仓库根目录 +`/home/xtkuang/Projects/cmvr/cmvr_edge_ai` 执行；从其他目录启动时，必须把 +配置中的模型等文件路径改为正确的绝对路径。 + ```bash -uv run --no-sync cmvr-edge-ai validate --config configs/smoke.yaml +uv run --no-sync cmvr-edge-ai validate \ + --config tests/fixtures/minimal_pipeline.yaml uv run --no-sync cmvr-edge-ai plugins uv run --no-sync cmvr-edge-ai run \ - --config configs/smoke.yaml \ + --config tests/fixtures/minimal_pipeline.yaml \ --log-level INFO \ --log-format text ``` @@ -113,7 +128,7 @@ bootstrap 支持以下环境： | Profile | 安装内容 | 命令 | |---|---|---| -| `core` | 框架核心和模拟 smoke/talk 链路 | `bash scripts/bootstrap.sh --profile core` | +| `core` | 框架核心、最小测试 fixture 和模拟 talk 链路 | `bash scripts/bootstrap.sh --profile core` | | `detection-cpu` | gRPC、HTTP、PyAV、Pillow 告警图片和固定版本 CPU YOLO；默认值 | `bash scripts/bootstrap.sh` | | `dev` | `detection-cpu` 加测试和 protobuf codegen 工具，并运行完整测试 | `bash scripts/bootstrap.sh --profile dev` | @@ -140,11 +155,16 @@ CLI 的四个子命令如下： `--pipeline` 可以重复传入。`run` 还支持 `--log-level` 和 `--log-format text|json`。配置错误退出码为 `2`，运行错误为 `1`，键盘中断为 `130`。 +生产配置 `configs/edge_ai.yaml` 在一个 YAML 中同时定义 `detection` 和 `talk`。部署时 +建议显式传 `--pipeline detection` 或 `--pipeline talk`，这样进程只加载并运行选中的 +链路；需要同进程运行两条链路时，可以重复传两个 `--pipeline`。如果完全省略 +`--pipeline`，运行时会启动配置中所有 `enabled: true` 的 Pipeline。 + ## 运行 PPE 检测链路 默认 bootstrap 就是当前 YAML 使用的 CPU 检测环境。它会从相邻的 `../cmvr-es` 读取 proto、用锁定的 `grpcio-tools` 生成 bindings，然后安装完整 -检测依赖并校验 smoke 和 PPE 配置： +检测依赖并校验最小测试 fixture 和统一配置中的 PPE 链路： ```bash cd /home/xtkuang/Projects/cmvr/cmvr_edge_ai @@ -173,13 +193,13 @@ uv sync --locked --only-group codegen 生成后应再次执行目标 profile 的 `uv sync --locked`，让可编辑安装识别新包；bootstrap 已经按这个顺序处理。 -在 `detect_server/pipeline.yaml` 中配置部署参数： +在 `configs/edge_ai.yaml` 的 `detection` Pipeline 中配置部署参数： ```yaml endpoints: cmvr_es: target: 127.0.0.1:50052 - platform: + ppe_alert_platform: base_url: http://127.0.0.1:8081 pipelines: @@ -194,15 +214,16 @@ pipelines: attach_frame: true inference_log_interval_s: 5 model_options: - weights: /home/xtkuang/Projects/cmvr/changan_robot/construction-ppe-yolov8/best.pt + weights: models/detection/construction-ppe-yolov8/v1/best.pt device: cpu repeat_gate: with: alert_image: enabled: true jpeg_quality: 85 - platform: + alert_platform: with: + endpoint: ppe_alert_platform failure_mode: log_and_drop ``` @@ -211,9 +232,9 @@ pipelines: ```bash uv run --no-sync cmvr-edge-ai models uv run --no-sync cmvr-edge-ai validate \ - --config detect_server/pipeline.yaml \ + --config configs/edge_ai.yaml \ --pipeline detection -uv run --no-sync cmvr-edge-ai run --config detect_server/pipeline.yaml \ +uv run --no-sync cmvr-edge-ai run --config configs/edge_ai.yaml \ --pipeline detection \ --log-level INFO \ --log-format json @@ -227,6 +248,19 @@ inference，说明相机或 decoder 尚未把帧送到模型；inference 中 det 当前阈值下没有命中。短时调试可设为 `1` 秒，生产环境可设为 `30`～`60` 秒，省略则关闭 周期推理日志。这里使用标准日志而不是裸 `print`，因此与 `--log-format json` 兼容。 +如果需要直接观察每次推理对应的画框图像，使用独立的 OpenCV Demo。它连接同一个 +cmvr-es gRPC CameraService，但不经过重复触发规则，也不会访问 HTTP 平台： + +```bash +uv run --no-sync python detect_server/show_detections.py \ + --config configs/debug/detection_viewer.yaml \ + --pipeline detection_show \ + --log-level INFO +``` + +运行前在 `configs/debug/detection_viewer.yaml` 中配置远端地址、`device_id` 和模型权重；按 `q` 或 `Esc` +退出。详细说明见 [detect_server/README.md](detect_server/README.md#实时画框-demo)。 + 相机连接器会在每次首次连接或重连时先发 `CameraService.StartCamera`，收到成功反馈后 才建立 `GetRGBImageStream`。终端会依次出现 `camera start requested/succeeded`、 `camera stream opening`、`camera stream first frame` 和周期性的 `camera stream progress`。 @@ -243,12 +277,19 @@ profile。项目保留了不绑定 CPU index 的 `yolo` extra 作为设备专用 GPU 部署前应为目标设备建立单独的 uv source/lock（或使用 NVIDIA 容器），再把 YAML 中的 `device` 改为 `cuda:0`；不要只改 YAML 就认为 CUDA 环境已经就绪。 -`detection.model@1` 根据 `model` 从 `DetectionModelRegistry` 解析模型。`detect_labels` 只选择需要检测的标签，省略时检测注册模型的全部标签；`confidence` 是全局阈值，也可以用 `label_confidence` 为个别标签覆盖。当前内置 `construction-ppe-yolov8@1` 的 19 个标签和 `ppe-6classes-yolov8n@1` 的 6 个标签都可以通过 `cmvr-edge-ai models` 查看。`attach_frame: true` 让检测结果临时携带对应的解码帧，供后续告警节点使用；因此原 detector 到 repeat gate 的队列应保持较小，避免堆积未压缩图像。 +`detection.model@1` 根据 `model` 从 `DetectionModelRegistry` 解析模型。`detect_labels` 只选择需要检测的标签，省略时检测注册模型的全部标签；`confidence` 是全局阈值，也可以用 `label_confidence` 为个别标签覆盖。当前内置 `construction-ppe-yolov8@1` 的 19 个标签和 `ppe-6classes-yolov8n@1` 的 6 个标签都可以通过 `cmvr-edge-ai models` 查看。两个模型制品和独立 model card 位于仓库内： + +- [Construction PPE YOLOv8 v1](models/detection/construction-ppe-yolov8/v1/README.md)：包含正向 PPE、`No-*` 违规类和施工现场设备类； +- [PPE YOLOv8n 6 Classes v1](models/detection/ppe-6classes-yolov8n/v1/README.md)：轻量的六类正向装备检测模型。 + +注册 ID 中的 `@1` 与制品目录的 `v1` 对应；这是项目的版本组织约定，实际 +`weights` 路径仍由部署 YAML 显式指定。标签顺序、训练指标、局限和许可声明以上述 +model card 为准。`attach_frame: true` 让检测结果临时携带对应的解码帧，供后续告警节点使用；因此原 detector 到 repeat gate 的队列应保持较小，避免堆积未压缩图像。 六类模型的标签是 `Gloves`、`Vest`、`goggles`、`helmet`、`mask` 和 `safety_shoe`， 语义是“画面中检测到了该装备”，不是“人员缺少该装备”。它没有 `Person` 或 `No-*` 类，也没有人员与装备关联能力，因此不能只靠配置推断某个人未佩戴 PPE。该模型当前仅 -注册、未被 `detect_server/pipeline.yaml` 引用；需要恢复第二分支时，应同时配置 detector、 +注册、未被 `configs/edge_ai.yaml` 的 `detection` Pipeline 引用；需要恢复第二分支时，应同时配置 detector、 8082 endpoint、HTTP Sink 和两条关联 edge。 若以后恢复双模型配置，应从 decoder 输出端口 fan-out，让两个 detector 共享同一个相机 @@ -279,8 +320,12 @@ GPU 部署前应为目标设备建立单独的 uv source/lock（或使用 NVIDIA 对话占位链路不依赖音频 proto： ```bash -uv run --no-sync cmvr-edge-ai validate --config talk_server/pipeline.yaml -uv run --no-sync cmvr-edge-ai run --config talk_server/pipeline.yaml +uv run --no-sync cmvr-edge-ai validate \ + --config configs/edge_ai.yaml \ + --pipeline talk +uv run --no-sync cmvr-edge-ai run \ + --config configs/edge_ai.yaml \ + --pipeline talk ``` ## 配置最小示例 @@ -324,6 +369,8 @@ v1 支持五个 `qos.profile`，并在编译期约束其溢出策略：编码 H2 ## 注册新的检测模型 检测模型和 DAG 插件是两层注册：流水线固定使用通用的 `detection.model@1`，具体模型通过 `DetectionModelRegistry` 注册 `DetectionModelSpec`。每个 spec 必须给出版本化 `model_id`、面向运维的 `name`、有序且唯一的 `supported_labels`、`backend` 和 factory。factory 返回实现 `load/predict/close` 的 `DetectionModel`；部署 YAML 中的 `model_options` 原样交给它。第三方模型包可以使用 `cmvr_edge_ai.detection_models` entry point 发布 spec 或注册回调。安装后先执行 `cmvr-edge-ai models`，再让配置引用其中的精确模型 ID。 +仓库自带制品统一放在 `models/detection//vN/`，并在每个版本目录保存 +`README.md` model card。新的 `model_id` 尾部 `@N` 应与制品目录 `vN` 保持一致。 模型实际输出的标签仍会在通用 Operator 边界二次校验和过滤；模型返回未注册标签会让节点失败。直接相连的重复规则若引用了 detector 没有选择的标签，也会在 `validate` 阶段被编译器拒绝。 diff --git a/configs/debug/detection_viewer.yaml b/configs/debug/detection_viewer.yaml new file mode 100644 index 0000000..996c31c --- /dev/null +++ b/configs/debug/detection_viewer.yaml @@ -0,0 +1,97 @@ +api_version: cmvr.edge.ai/v1 + +runtime: + # Decoder, YOLO and viewer drawing share this bounded application pool. + thread_workers: 3 + shutdown_timeout_s: 8 + +endpoints: + cmvr_es: + transport: grpc + # Remote cmvr-es address. Keep this aligned with the robot being viewed. + target: 192.168.0.119:50052 + tls: false + timeout_s: 5 + options: + max_receive_mb: 32 + +pipelines: + detection_show: + enabled: true + nodes: + camera: + uses: cmvr.grpc.camera_rgb_stream@1 + with: + endpoint: cmvr_es + device_id: wrist_cam + pixel_format: BGR8 + reconnect: true + reconnect_initial_s: 0.5 + reconnect_max_s: 10 + stream_log_interval_s: 5 + + decoder: + uses: media.video_decoder.pyav@1 + + detector: + uses: detection.model@1 + with: + model: construction-ppe-yolov8@1 + detect_labels: + - No-Boots + - No-Ear-Protection + - No-Glass + - No-Glove + - No-Helmet + - No-Mask + - No-Vest + confidence: 0.50 + # A 30 FPS stream starts at most 10 model inferences each second. + max_fps: 10 + inference_log_interval_s: 5 + # Required so the viewer receives the exact decoded inference frame. + attach_frame: true + model_options: + # Repository-relative path; launch the viewer from the repository root. + weights: models/detection/construction-ppe-yolov8/v1/best.pt + device: cpu + imgsz: 640 + iou: 0.70 + half: false + max_det: 100 + + viewer: + uses: demo.opencv_detection_viewer@1 + with: + window_name: CMVR PPE Detection + window_width: 1280 + window_height: 720 + wait_key_ms: 1 + box_thickness: 2 + font_scale: 0.6 + show_stats: true + + edges: + # H264/H265 packets must remain contiguous until decoding. + - from: camera.frames + to: decoder.frames + qos: + profile: video_contiguous + capacity: 8 + overflow: block + + # Keep only the latest decoded frame while YOLO is busy. + - from: decoder.frames + to: detector.frames + qos: + profile: realtime_latest + capacity: 1 + overflow: drop_oldest + + # A slow GUI must not accumulate raw frames or stale detection results. + - from: detector.detections + to: viewer.input + qos: + profile: realtime_latest + capacity: 1 + overflow: drop_oldest diff --git a/detect_server/pipeline.yaml b/configs/edge_ai.yaml similarity index 80% rename from detect_server/pipeline.yaml rename to configs/edge_ai.yaml index 2ce7667..797159b 100644 --- a/detect_server/pipeline.yaml +++ b/configs/edge_ai.yaml @@ -8,16 +8,16 @@ endpoints: cmvr_es: transport: grpc # cmvr-es gRPC address. Change this value for each deployed robot. - target: 192.168.0.102:50052 + target: 192.168.0.119:50052 tls: false timeout_s: 5 - options: + options: max_receive_mb: 32 - platform: + ppe_alert_platform: transport: http # Violation-alert platform HTTP base URL. - base_url: http://127.0.0.1:8081 + base_url: http://192.168.0.222:13080 timeout_s: 3 pipelines: @@ -65,8 +65,9 @@ pipelines: attach_frame: true model_options: # Model artifact and inference device are deployment configuration, - # not process environment requirements. - weights: /home/xtkuang/Projects/cmvr/changan_robot/construction-ppe-yolov8/best.pt + # not process environment requirements. This repository-relative + # path requires launching cmvr-edge-ai from the repository root. + weights: models/detection/construction-ppe-yolov8/v1/best.pt device: cpu imgsz: 640 iou: 0.70 @@ -133,10 +134,10 @@ pipelines: min_confidence: 0.50 scope: source - platform: + alert_platform: uses: platform.http_json_sink@1 with: - endpoint: platform + endpoint: ppe_alert_platform path: /v1/detection-alerts # Platform outages must not stop camera capture or inference. After # bounded retries, log a WARNING and drop only this report. @@ -173,8 +174,44 @@ pipelines: overflow: drop_oldest - from: repeat_gate.alerts - to: platform.input + to: alert_platform.input qos: profile: telemetry capacity: 64 overflow: block + + talk: + enabled: true + nodes: + audio_stream_placeholder: + # Replace with cmvr.grpc.microphone_audio_stream@1 when its proto lands. + uses: core.sequence_source@1 + with: + items: + - simulated-audio-chunk + schema_name: AudioChunk + schema_version: 1 + + dialogue_placeholder: + # The real chain will be VAD -> ASR -> dialogue -> TTS. + uses: core.passthrough@1 + + output: + uses: core.log_sink@1 + with: + logger: cmvr_edge_ai.talk + + edges: + - from: audio_stream_placeholder.output + to: dialogue_placeholder.input + qos: + profile: audio_contiguous + capacity: 16 + overflow: block + + - from: dialogue_placeholder.output + to: output.input + qos: + profile: request + capacity: 8 + overflow: block diff --git a/detect_server/README.md b/detect_server/README.md index 7c535fc..42a7b1c 100644 --- a/detect_server/README.md +++ b/detect_server/README.md @@ -1,7 +1,8 @@ # PPE 检测流水线 -`pipeline.yaml` 是当前可运行的园区施工安全装备检测链路。当前只启用 Construction -PPE 模型和 8081 告警平台；六类模型与 8082 模拟平台的实现保留但不实例化： +`configs/edge_ai.yaml` 中的 `detection` Pipeline 是当前可运行的园区施工安全装备检测 +链路。它与 `talk` Pipeline 共用一个部署 YAML，当前只启用 Construction PPE 模型和 +8081 告警平台；六类模型与 8082 模拟平台的实现保留但不实例化： ```text cmvr-es CameraService @@ -16,7 +17,7 @@ cmvr-es CameraService ## 安装与启动 从仓库根目录执行一键安装。默认 profile 安装锁定的 CPU 检测环境、生成 cmvr-es -bindings，并验证 smoke 与本检测配置： +bindings，并验证最小测试 fixture 与统一部署配置中的检测链路： ```bash cd /home/xtkuang/Projects/cmvr/cmvr_edge_ai @@ -36,13 +37,13 @@ bash scripts/bootstrap.sh \ wheel，不能直接复用 `detection-cpu` profile。手动组合依赖时必须显式增加 `--extra image`，不能只依赖 YOLO 间接安装 Pillow。 -直接编辑 `detect_server/pipeline.yaml` 中的部署参数： +直接编辑 `configs/edge_ai.yaml` 中 `detection` Pipeline 的部署参数： ```yaml endpoints: cmvr_es: target: 127.0.0.1:50052 - platform: + ppe_alert_platform: base_url: http://127.0.0.1:8081 pipelines: @@ -57,26 +58,32 @@ pipelines: attach_frame: true inference_log_interval_s: 5 model_options: - weights: /home/xtkuang/Projects/cmvr/changan_robot/construction-ppe-yolov8/best.pt + weights: models/detection/construction-ppe-yolov8/v1/best.pt device: cpu repeat_gate: with: alert_image: enabled: true jpeg_quality: 85 - platform: + alert_platform: with: + endpoint: ppe_alert_platform failure_mode: log_and_drop ``` +上述相对权重路径按进程启动时的当前工作目录（`cwd`）解析，不是按 +`configs/edge_ai.yaml` 所在目录解析。下面的 validate、run 和 Viewer 命令都应先 +`cd /home/xtkuang/Projects/cmvr/cmvr_edge_ai`；如果必须在其他 `cwd` 启动，请在 +YAML 中使用正确的绝对权重路径。 + 完成配置后启动，不需要再通过 shell `export` 传入这些值： ```bash uv run --no-sync cmvr-edge-ai models uv run --no-sync cmvr-edge-ai validate \ - -c detect_server/pipeline.yaml \ + -c configs/edge_ai.yaml \ --pipeline detection -uv run --no-sync cmvr-edge-ai run -c detect_server/pipeline.yaml \ +uv run --no-sync cmvr-edge-ai run -c configs/edge_ai.yaml \ --pipeline detection \ --log-level INFO \ --log-format json @@ -102,16 +109,83 @@ uv run --no-sync cmvr-edge-ai run -c detect_server/pipeline.yaml \ 平台接受 `POST /v1/detection-alerts`。默认 profile 下 `model_options.device` 应设为 `cpu`；只有完成设备专用的 CUDA/Jetson PyTorch 环境适配后，才能改为 `cuda:0` 等值。 +## 实时画框 Demo + +`configs/debug/detection_viewer.yaml` 和 `show_detections.py` 提供一个不访问 HTTP 平台的 +独立调试链路： + +```text +cmvr-es CameraService -> PyAV decoder -> YOLO detector -> OpenCV window +``` + +先编辑 `configs/debug/detection_viewer.yaml` 中的远端 cmvr-es 地址、相机 ID 和权重路径： + +```yaml +endpoints: + cmvr_es: + target: 192.168.0.119:50052 + +pipelines: + detection_show: + nodes: + camera: + with: + device_id: wrist_cam + detector: + with: + max_fps: 10 + model_options: + weights: /absolute/path/to/best.pt + device: cpu +``` + +`max_fps: 10` 表示最多每秒执行 10 次推理；视频流更快时，中间的已解码帧通过 +`realtime_latest + drop_oldest` 丢弃，以保持低延迟。`attach_frame: true` 已在 Demo 配置 +中启用，viewer 因而能拿到与本次推理严格对应的原图并绘制 bounding box。 + +只检查配置和插件连线，不连接相机、不加载模型、也不创建窗口： + +```bash +uv run --no-sync python detect_server/show_detections.py \ + --config configs/debug/detection_viewer.yaml \ + --validate-only +``` + +启动实时显示： + +```bash +uv run --no-sync python detect_server/show_detections.py \ + --config configs/debug/detection_viewer.yaml \ + --pipeline detection_show \ + --log-level INFO \ + --log-format json +``` + +相机 Source 会先调用 `StartCamera`，成功后再建立 gRPC 视频流。窗口显示每个实际推理 +结果，即使本帧没有检测框也会刷新；按 `q`、`Q`、`Esc` 或关闭窗口可安全退出。这个 +Demo 直接订阅 detector 输出，刻意绕过 `repeat_gate` 和 HTTP Sink，因此只用于观察模型 +效果，不代表某条告警规则已满足。 + +OpenCV 窗口出现在运行命令的机器上。无桌面的边缘设备不能直接显示；通过 SSH 运行时 +需要启用 X11 转发并确保 `DISPLAY` 可用，否则程序会给出明确错误并退出。依赖缺失时先 +执行 `bash scripts/bootstrap.sh`。 + ## 模型与标签 `detection.model@1` 不绑定某一个框架；它通过 `DetectionModelRegistry` 查找配置中的 `model`。每个 `DetectionModelSpec` 注册版本化模型 ID、模型名称、backend 和有序 `supported_labels`。注册表当前包含两个内置模型，但 Pipeline 只引用第一个： -- `construction-ppe-yolov8@1`：19 类，包含原分支用于违规告警的 `No-*` 标签； -- `ppe-6classes-yolov8n@1`：`Gloves`、`Vest`、`goggles`、`helmet`、`mask`、 +- [`construction-ppe-yolov8@1`](../models/detection/construction-ppe-yolov8/v1/README.md)： + 对应制品目录 `v1`，19 类，包含用于违规告警的 `No-*` 标签； +- [`ppe-6classes-yolov8n@1`](../models/detection/ppe-6classes-yolov8n/v1/README.md)： + 对应制品目录 `v1`，包含 `Gloves`、`Vest`、`goggles`、`helmet`、`mask`、 `safety_shoe` 六个正向装备标签。 +模型 ID 尾部的 `@1` 与制品目录的 `v1` 对应；运行时不会根据 ID 自动拼接文件 +路径，仍由 YAML 中的 `model_options.weights` 显式指定。训练信息、完整标签 +顺序、性能、限制和许可信息请查看各自的 model card。 + 可用 `cmvr-edge-ai models` 核对 ID、名称、backend 和标签顺序。六类模型只表达 “检测到某件装备”，不包含 `Person` 或 `No-*` 类，也不执行人员/PPE 关联；所以它 不能直接判断某个人缺少装备。需要这种语义时，仍应增加人员检测、空间关联和缺失 diff --git a/detect_server/show_detections.py b/detect_server/show_detections.py new file mode 100644 index 0000000..ed17adc --- /dev/null +++ b/detect_server/show_detections.py @@ -0,0 +1,636 @@ +#!/usr/bin/env python3 +"""Display live cmvr-es detection results in a local OpenCV window. + +This is an isolated demo entrypoint. It registers a temporary viewer Sink and +does not change the production detection pipeline or its HTTP alert behavior. +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import math +import os +import signal +import sys +from collections.abc import Callable, Mapping +from concurrent.futures import Executor +from importlib import import_module +from pathlib import Path +from time import monotonic +from typing import Any + +from pydantic import ValidationError + +from cmvr_edge_ai.application import ( + EdgeAIApplication, + create_default_model_registry, + create_default_registry, + validate_application, +) +from cmvr_edge_ai.compiler import PipelineCompileError +from cmvr_edge_ai.config import AppConfig, ConfigLoadError, load_config +from cmvr_edge_ai.contracts import Detection, DetectionResult, ImageFrame +from cmvr_edge_ai.core import ComponentContext, Envelope, Sink +from cmvr_edge_ai.observability import configure_logging +from cmvr_edge_ai.plugins import PluginKind, PluginRegistry, PluginSpec +from cmvr_edge_ai.workers import run_blocking + + +_LOGGER = logging.getLogger("cmvr_edge_ai.demo.detection_viewer") +_DEFAULT_CONFIG = ( + Path(__file__).resolve().parents[1] + / "configs" + / "debug" + / "detection_viewer.yaml" +) +_VIEWER_PLUGIN_ID = "demo.opencv_detection_viewer@1" + + +class DetectionViewerError(RuntimeError): + """Raised when a detection frame cannot be displayed safely.""" + + +class OpenCvDetectionViewerSink(Sink): + """Render ``DetectionResult`` boxes and show the corresponding source frame.""" + + _PARAM_KEYS = frozenset( + { + "window_name", + "window_width", + "window_height", + "wait_key_ms", + "box_thickness", + "font_scale", + "show_stats", + } + ) + + def __init__( + self, + node_id: str, + params: Mapping[str, Any], + *, + request_stop: Callable[[], None], + module_loader: Callable[[str], Any] = import_module, + ) -> None: + unknown = set(params) - self._PARAM_KEYS + if unknown: + raise ValueError( + "unknown detection viewer parameter(s): " + + ", ".join(sorted(str(value) for value in unknown)) + ) + if not callable(request_stop): + raise TypeError("request_stop must be callable") + + self._node_id = node_id + self._window_name = _non_empty_string( + params.get("window_name", "CMVR PPE Detection"), + "window_name", + ) + self._window_width = _bounded_int( + params.get("window_width", 1280), + "window_width", + minimum=1, + maximum=16384, + ) + self._window_height = _bounded_int( + params.get("window_height", 720), + "window_height", + minimum=1, + maximum=16384, + ) + self._wait_key_ms = _bounded_int( + params.get("wait_key_ms", 1), + "wait_key_ms", + minimum=1, + maximum=20, + ) + self._box_thickness = _bounded_int( + params.get("box_thickness", 2), + "box_thickness", + minimum=1, + maximum=20, + ) + self._font_scale = _bounded_float( + params.get("font_scale", 0.6), + "font_scale", + minimum=0.1, + maximum=5.0, + ) + self._show_stats = _strict_bool( + params.get("show_stats", True), + "show_stats", + ) + self._request_stop = request_stop + self._module_loader = module_loader + self._executor: Executor | None = None + self._cv2: Any = None + self._numpy: Any = None + self._window_open = False + self._stop_requested = False + self._event_pump_task: asyncio.Task[None] | None = None + self._window_seen_visible = False + self._frames_shown = 0 + self._last_frame_at: float | None = None + self._display_fps: float | None = None + + async def setup(self, context: ComponentContext) -> None: + executor = context.metadata.get("thread_executor") + if executor is not None and not isinstance(executor, Executor): + raise TypeError("component context thread_executor must be an Executor") + self._executor = executor + + if sys.platform.startswith("linux") and not _linux_display_available(os.environ): + raise DetectionViewerError( + "OpenCV viewer needs a graphical session; DISPLAY and " + "WAYLAND_DISPLAY are both unset. Run it on the desktop or use " + "SSH X11 forwarding." + ) + try: + cv2 = self._module_loader("cv2") + numpy = self._module_loader("numpy") + except ImportError as exc: + raise DetectionViewerError( + "OpenCV viewer dependencies are missing; run " + "'bash scripts/bootstrap.sh' first" + ) from exc + + gui_backend = _opencv_gui_backend(cv2.getBuildInformation()) + if gui_backend.upper() in {"NONE", "NO"}: + raise DetectionViewerError( + "installed OpenCV has no GUI backend; install opencv-python " + "instead of opencv-python-headless" + ) + + self._cv2 = cv2 + self._numpy = numpy + try: + cv2.namedWindow(self._window_name, cv2.WINDOW_NORMAL) + self._window_open = True + cv2.resizeWindow( + self._window_name, + self._window_width, + self._window_height, + ) + except Exception as exc: + if self._window_open: + try: + cv2.destroyWindow(self._window_name) + except Exception: + pass + self._window_open = False + raise DetectionViewerError( + "OpenCV could not create the viewer window; verify the local " + "desktop session and DISPLAY configuration" + ) from exc + _LOGGER.info( + "detection viewer opened node=%s window=%s gui_backend=%s " + "quit_keys=q,esc", + self._node_id, + self._window_name, + gui_backend, + ) + + async def start(self) -> None: + if not self._window_open or self._cv2 is None: + raise RuntimeError("detection viewer has not been set up") + if self._event_pump_task is not None: + raise RuntimeError("detection viewer has already been started") + self._event_pump_task = asyncio.create_task( + self._pump_window_events(), + name=f"detection-viewer-events:{self._node_id}", + ) + + async def consume( + self, + envelope: Envelope[Any], + input_port: str = "input", + ) -> None: + del input_port + if self._stop_requested: + return + if not self._window_open or self._cv2 is None or self._numpy is None: + raise RuntimeError("detection viewer has not been set up") + result = envelope.payload + if not isinstance(result, DetectionResult): + raise TypeError( + f"{self._node_id} expected DetectionResult, " + f"got {type(result).__name__}" + ) + frame = result.source_frame + if frame is None: + raise DetectionViewerError( + "DetectionResult has no source_frame; set detector " + "attach_frame: true in the demo config" + ) + + now = monotonic() + if self._last_frame_at is not None and now > self._last_frame_at: + instantaneous_fps = 1.0 / (now - self._last_frame_at) + self._display_fps = ( + instantaneous_fps + if self._display_fps is None + else self._display_fps * 0.85 + instantaneous_fps * 0.15 + ) + self._last_frame_at = now + header = None + if self._show_stats: + display_fps = self._display_fps or 0.0 + model_name = result.model_name or result.model_id + header = ( + f"{model_name} | boxes={len(result.detections)} | " + f"inference={result.inference_ms:.1f} ms | display={display_fps:.1f} FPS" + ) + + image = await run_blocking( + _render_detection_frame, + frame, + result.detections, + cv2_module=self._cv2, + numpy_module=self._numpy, + box_thickness=self._box_thickness, + font_scale=self._font_scale, + header=header, + executor=self._executor, + ) + self._cv2.imshow(self._window_name, image) + self._frames_shown += 1 + + async def stop(self) -> None: + if not self._window_open: + return + self._window_open = False + if self._event_pump_task is not None: + self._event_pump_task.cancel() + await asyncio.gather(self._event_pump_task, return_exceptions=True) + self._event_pump_task = None + try: + self._cv2.destroyWindow(self._window_name) + except Exception: + _LOGGER.warning( + "detection viewer window cleanup failed node=%s window=%s", + self._node_id, + self._window_name, + exc_info=True, + ) + _LOGGER.info( + "detection viewer stopped node=%s frames_shown=%s", + self._node_id, + self._frames_shown, + ) + + async def _pump_window_events(self) -> None: + """Keep the GUI responsive even while the camera produces no frames.""" + + try: + while self._window_open: + try: + key = int(self._cv2.waitKey(self._wait_key_ms)) & 0xFF + if key in {27, ord("q"), ord("Q")}: + self._request_demo_stop("keyboard") + return + visible = float( + self._cv2.getWindowProperty( + self._window_name, + self._cv2.WND_PROP_VISIBLE, + ) + ) + except Exception: + # Visibility queries are optional in some GUI backends. + visible = -1.0 + + if visible >= 1.0: + self._window_seen_visible = True + elif visible == 0.0 or ( + visible < 0.0 and self._window_seen_visible + ): + self._request_demo_stop("window_closed") + return + + # waitKey processes native events; this small cooperative pause + # prevents an idle, frame-less stream from spinning one CPU core. + await asyncio.sleep(max(0.01, self._wait_key_ms / 1000.0)) + except asyncio.CancelledError: + raise + + def _request_demo_stop(self, reason: str) -> None: + if self._stop_requested: + return + self._stop_requested = True + _LOGGER.info( + "detection viewer stop requested node=%s reason=%s frames_shown=%s", + self._node_id, + reason, + self._frames_shown, + ) + self._request_stop() + + +def _render_detection_frame( + frame: ImageFrame, + detections: tuple[Detection, ...], + *, + cv2_module: Any, + numpy_module: Any, + box_thickness: int, + font_scale: float, + header: str | None, +) -> Any: + """Copy one packed frame and draw validated boxes into a BGR ndarray.""" + + _validate_display_frame(frame) + image = ( + numpy_module.frombuffer(frame.data, dtype=numpy_module.uint8) + .reshape((frame.height, frame.width, 3)) + .copy() + ) + if frame.pixel_format.strip().upper() == "RGB8": + image = cv2_module.cvtColor(image, cv2_module.COLOR_RGB2BGR) + + if header: + cv2_module.rectangle( + image, + (0, 0), + (frame.width - 1, min(frame.height - 1, 30)), + (24, 24, 24), + -1, + ) + cv2_module.putText( + image, + header, + (8, min(frame.height - 1, 21)), + cv2_module.FONT_HERSHEY_SIMPLEX, + 0.55, + (255, 255, 255), + 1, + cv2_module.LINE_AA, + ) + + # Draw evidence after the status bar so boxes at the top of the image are + # never hidden behind presentation-only statistics. + for detection in detections: + if not isinstance(detection, Detection): + raise DetectionViewerError( + "DetectionResult.detections must contain Detection instances" + ) + box = _clipped_box(detection, frame.width, frame.height) + if box is None: + continue + color = _label_color_bgr(detection.label) + x_min, y_min, x_max, y_max = box + cv2_module.rectangle( + image, + (x_min, y_min), + (x_max, y_max), + color, + box_thickness, + ) + label = f"{detection.label} {detection.confidence:.2f}" + (text_width, text_height), baseline = cv2_module.getTextSize( + label, + cv2_module.FONT_HERSHEY_SIMPLEX, + font_scale, + 1, + ) + text_bottom = max(text_height + baseline + 4, y_min) + background_top = max(0, text_bottom - text_height - baseline - 6) + background_right = min(frame.width - 1, x_min + text_width + 6) + cv2_module.rectangle( + image, + (x_min, background_top), + (background_right, text_bottom), + color, + -1, + ) + cv2_module.putText( + image, + label, + (x_min + 3, max(text_height + 1, text_bottom - baseline - 3)), + cv2_module.FONT_HERSHEY_SIMPLEX, + font_scale, + (255, 255, 255), + 1, + cv2_module.LINE_AA, + ) + + return image + + +def _validate_display_frame(frame: ImageFrame) -> None: + if not isinstance(frame, ImageFrame): + raise DetectionViewerError( + f"expected ImageFrame, got {type(frame).__name__}" + ) + if not isinstance(frame.codec, str): + raise DetectionViewerError("frame codec must be a string") + if frame.is_encoded: + raise DetectionViewerError("viewer requires a decoded ImageFrame") + if not isinstance(frame.pixel_format, str): + raise DetectionViewerError("frame pixel_format must be a string") + pixel_format = frame.pixel_format.strip().upper() + if pixel_format not in {"BGR8", "RGB8"}: + raise DetectionViewerError( + f"viewer supports BGR8 or RGB8, got {frame.pixel_format!r}" + ) + if ( + isinstance(frame.width, bool) + or not isinstance(frame.width, int) + or frame.width < 1 + or isinstance(frame.height, bool) + or not isinstance(frame.height, int) + or frame.height < 1 + ): + raise DetectionViewerError("frame dimensions must be positive integers") + if not isinstance(frame.data, bytes): + raise DetectionViewerError("packed frame buffer must be bytes") + expected_size = frame.width * frame.height * 3 + if len(frame.data) != expected_size: + raise DetectionViewerError( + f"packed frame has {len(frame.data)} bytes; expected {expected_size}" + ) + + +def _clipped_box( + detection: Detection, + width: int, + height: int, +) -> tuple[int, int, int, int] | None: + values = ( + detection.box.x_min, + detection.box.y_min, + detection.box.x_max, + detection.box.y_max, + ) + if any( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + for value in values + ): + return None + x_min, y_min, x_max, y_max = (float(value) for value in values) + if x_max <= x_min or y_max <= y_min: + return None + left = min(max(int(math.floor(x_min)), 0), width - 1) + top = min(max(int(math.floor(y_min)), 0), height - 1) + right = min(max(int(math.ceil(x_max)), 0), width - 1) + bottom = min(max(int(math.ceil(y_max)), 0), height - 1) + if right <= left or bottom <= top: + return None + return left, top, right, bottom + + +def _label_color_bgr(label: str) -> tuple[int, int, int]: + seed = sum((index + 1) * byte for index, byte in enumerate(label.encode("utf-8"))) + return ( + 64 + (seed * 11) % 192, + 64 + (seed * 5) % 192, + 64 + seed % 192, + ) + + +def _linux_display_available(environment: Mapping[str, str]) -> bool: + return bool(environment.get("DISPLAY") or environment.get("WAYLAND_DISPLAY")) + + +def _opencv_gui_backend(build_information: str) -> str: + for line in str(build_information).splitlines(): + stripped = line.strip() + if stripped.upper().startswith("GUI:"): + return stripped.split(":", maxsplit=1)[1].strip() or "unknown" + return "unknown" + + +def _non_empty_string(value: Any, name: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{name} must be a non-empty string") + return value.strip() + + +def _bounded_int(value: Any, name: str, *, minimum: int, maximum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + if not minimum <= value <= maximum: + raise ValueError(f"{name} must be between {minimum} and {maximum}") + return value + + +def _bounded_float( + value: Any, + name: str, + *, + minimum: float, + maximum: float, +) -> float: + if isinstance(value, bool): + raise ValueError(f"{name} must be a number") + try: + parsed = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be a number") from exc + if not math.isfinite(parsed) or not minimum <= parsed <= maximum: + raise ValueError(f"{name} must be between {minimum} and {maximum}") + return parsed + + +def _strict_bool(value: Any, name: str) -> bool: + if type(value) is not bool: + raise ValueError(f"{name} must be a boolean") + return value + + +def _build_registry(request_stop: Callable[[], None]) -> PluginRegistry: + model_registry = create_default_model_registry() + registry = create_default_registry(model_registry=model_registry) + registry.register( + PluginSpec( + plugin_id=_VIEWER_PLUGIN_ID, + kind=PluginKind.SINK, + factory=lambda node_id, params: OpenCvDetectionViewerSink( + node_id, + params, + request_stop=request_stop, + ), + inputs={"input": "DetectionResult/v1"}, + description="Show detection source frames with bounding boxes in OpenCV", + tags=frozenset({"demo", "visualization"}), + ) + ) + return registry + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Show live cmvr-es YOLO detections in an OpenCV window" + ) + parser.add_argument("--config", "-c", type=Path, default=_DEFAULT_CONFIG) + parser.add_argument("--pipeline", default="detection_show") + parser.add_argument("--log-level", default="INFO") + parser.add_argument("--log-format", choices=("text", "json"), default="text") + parser.add_argument( + "--validate-only", + action="store_true", + help="validate the demo graph without opening a camera or GUI window", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + configure_logging(args.log_level, args.log_format) + try: + config = load_config(args.config) + if args.validate_only: + validate_application(config, _build_registry(lambda: None), (args.pipeline,)) + print(f"demo configuration is valid; pipeline: {args.pipeline}") + return 0 + return asyncio.run(_run_demo(config, args.pipeline)) + except (ConfigLoadError, ValidationError, PipelineCompileError, ValueError) as exc: + print(f"configuration error: {exc}", file=sys.stderr) + return 2 + except KeyboardInterrupt: + return 130 + except Exception as exc: + print(f"runtime error: {exc}", file=sys.stderr) + return 1 + + +async def _run_demo(config: AppConfig, pipeline_id: str) -> int: + stop_event = asyncio.Event() + application = EdgeAIApplication(config, _build_registry(stop_event.set)) + loop = asyncio.get_running_loop() + for signum in (signal.SIGINT, signal.SIGTERM): + try: + loop.add_signal_handler(signum, stop_event.set) + except NotImplementedError: + pass + + await application.start((pipeline_id,)) + _LOGGER.info( + "detection viewer demo running pipeline=%s; press q or Esc in the window to stop", + pipeline_id, + ) + wait_task = asyncio.create_task(application.wait(), name="demo-application-wait") + stop_task = asyncio.create_task(stop_event.wait(), name="demo-stop-request") + try: + done, _ = await asyncio.wait( + (wait_task, stop_task), + return_when=asyncio.FIRST_COMPLETED, + ) + if wait_task in done: + await wait_task + else: + await application.stop(graceful=True) + wait_task.cancel() + await asyncio.gather(wait_task, return_exceptions=True) + finally: + stop_task.cancel() + await asyncio.gather(stop_task, return_exceptions=True) + await application.stop(graceful=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/architecture.md b/docs/architecture.md index 73b5cb6..f3a33c3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -224,6 +224,11 @@ HTTP 是否使用 TLS 由 `base_url` 的 `https://` scheme 决定，证书验证 | `nodes` | 无 | 至少一个节点 | | `edges` | `[]` | 有向连接列表 | +仓库的生产配置 `configs/edge_ai.yaml` 在同一个 YAML 中定义 `detection` 和 `talk`。 +CLI 的 `--pipeline` 可以重复传入：显式传 `--pipeline detection` 或 `--pipeline talk` 时 +只编译并运行所选链路；省略该参数时会启动所有 `enabled: true` 的 Pipeline。生产部署 +通常应显式选择 Pipeline，需要共享同一进程和网络客户端时才同时选择两条链路。 + 不要在活跃 Pipeline 中保留 `enabled: false` 节点；当前编译器会直接拒绝。要暂时关闭逻辑，请禁用整个 Pipeline 或从图和配置中移除该节点。 ### 3.5 `nodes.` @@ -351,6 +356,18 @@ with: `cmvr-edge-ai models` 输出当前 model registry 的 ID、name、backend 和 labels。第三方包可在 `cmvr_edge_ai.detection_models` entry point 中暴露一个 `DetectionModelSpec` 或注册回调。当前内置 `construction-ppe-yolov8@1`（19 类）和 `ppe-6classes-yolov8n@1`（`Gloves/Vest/goggles/helmet/mask/safety_shoe`）；YOLO adapter 在加载时严格比较 checkpoint `model.names` 与各自注册的标签及顺序，避免错误的类别编号继续运行。 +内置检测模型制品按 `models/detection//vN/` 组织，每个版本目录同时 +保存权重和独立 model card： + +- [Construction PPE YOLOv8 v1](../models/detection/construction-ppe-yolov8/v1/README.md)； +- [PPE YOLOv8n 6 Classes v1](../models/detection/ppe-6classes-yolov8n/v1/README.md)。 + +`DetectionModelSpec.model_id` 尾部的 `@N` 与制品目录的 `vN` 对应，例如 +`construction-ppe-yolov8@1` 对应 `construction-ppe-yolov8/v1/`。这是注册表与制品 +库的版本约定，运行时不会由模型 ID 自动推导权重路径；部署配置仍必须 +显式给出 `model_options.weights`。标签、训练来源、评估、局限和许可信息由每个 +版本目录的 model card 维护，架构文档只定义制品与运行时的边界。 + `detection.model@1` 参数： ```yaml @@ -364,11 +381,15 @@ with: inference_log_interval_s: 5 attach_frame: true model_options: - weights: /home/xtkuang/Projects/cmvr/changan_robot/construction-ppe-yolov8/best.pt + weights: models/detection/construction-ppe-yolov8/v1/best.pt device: cpu imgsz: 640 ``` +`weights` 等相对文件路径按进程启动时的当前工作目录（`cwd`）解析，不是按 +YAML 文件的所在目录解析。仓库内配置和文档命令以仓库根目录为 `cwd`； +从其他目录启动时应使用绝对路径或在部署前将路径正规化。 + - `detect_labels` 省略时选择模型注册的全部标签；显式空列表、重复或未知标签会失败； - `confidence` 是全局阈值，`label_confidence` 可逐标签覆盖；backend 接收所有选中标签中的最低阈值，通用 Operator 再逐框做严格后过滤； - `max_fps` 是推理启动频率上限，跳过的帧不会产生 `DetectionResult`； @@ -767,11 +788,15 @@ AI 结果 -> Policy -> RobotCommand/v1 ```bash uv run --no-sync cmvr-edge-ai models -uv run --no-sync cmvr-edge-ai validate -c detect_server/pipeline.yaml --pipeline detection -uv run --no-sync cmvr-edge-ai run -c detect_server/pipeline.yaml \ +uv run --no-sync cmvr-edge-ai validate -c configs/edge_ai.yaml --pipeline detection +uv run --no-sync cmvr-edge-ai run -c configs/edge_ai.yaml \ --pipeline detection \ --log-level INFO \ --log-format json +uv run --no-sync cmvr-edge-ai run -c configs/edge_ai.yaml \ + --pipeline talk \ + --log-level INFO \ + --log-format text ``` 当前日志可输出文本或单行 JSON。日志中不要写入音频原始数据、图像 base64、认证 metadata 或用户隐私内容；生产插件应只记录 trace ID、schema、耗时、尺寸、丢弃计数和经过脱敏的错误信息。 diff --git a/models/README.md b/models/README.md new file mode 100644 index 0000000..c0541db --- /dev/null +++ b/models/README.md @@ -0,0 +1,53 @@ +# 模型资产目录 + +本目录保存 cmvr-edge-ai 在边缘端部署时使用的模型资产和模型说明。它与 +`src/cmvr_edge_ai/detection/models/` 的 Python 代码职责不同： + +- `models/` 保存权重、版本和模型卡； +- `src/cmvr_edge_ai/detection/models/` 保存 backend adapter 与模型注册代码； +- `configs/` 决定某条 Pipeline 选择哪个注册模型和哪份权重。 + +目前的资产类型： + +- [Detection 模型](detection/README.md) + +## 目录约定 + +模型资产使用以下结构： + +```text +models/ +└── detection/ + └── / + └── v/ + ├── README.md + └── +``` + +注册 ID 中的数字版本与目录版本一一对应。例如： + +```text +construction-ppe-yolov8@1 + │ + └── models/detection/construction-ppe-yolov8/v1/ +``` + +发布新权重时应新增版本目录和新的注册 ID，不能直接覆盖已经部署的权重。模型卡至少应 +记录有序标签、输入要求、运行参数、评估指标、限制、许可证、来源和权重 SHA256。 + +## 大文件管理 + +Detection 的 `.pt` 权重使用 Git LFS。克隆仓库后若权重尚未下载，执行： + +```bash +git lfs install +git lfs pull +``` + +将新权重加入仓库前，先核对模型卡中的 SHA256： + +```bash +sha256sum models/detection//v/ +``` + +不要把训练数据集、训练缓存、原模型仓库的 `.git` 目录或无关评估产物放入本目录。 diff --git a/models/detection/README.md b/models/detection/README.md new file mode 100644 index 0000000..badbaee --- /dev/null +++ b/models/detection/README.md @@ -0,0 +1,48 @@ +# Detection 模型资产 + +本目录集中管理 cmvr-edge-ai 已注册的目标检测模型。Pipeline 仍统一使用 +`detection.model@1` 节点；节点的 `with.model` 选择注册 ID， +`with.model_options.weights` 选择本目录中的具体权重。 + +## 当前模型 + +| 注册模型 ID | 版本目录 | Backend | 标签数 | 标签语义 | +|---|---|---|---:|---| +| `construction-ppe-yolov8@1` | [construction-ppe-yolov8/v1](construction-ppe-yolov8/v1/README.md) | `ultralytics-yolo` | 19 | PPE、PPE 缺失违规及部分现场设备 | +| `ppe-6classes-yolov8n@1` | [ppe-6classes-yolov8n/v1](ppe-6classes-yolov8n/v1/README.md) | `ultralytics-yolo` | 6 | 画面中实际出现的六类 PPE | + +可通过以下命令查看运行时注册信息及有序标签： + +```bash +uv run --no-sync cmvr-edge-ai models +``` + +## 配置规则 + +相对权重路径按启动进程的当前工作目录解析。本文档中的示例假定命令从仓库根目录执行： + +```yaml +nodes: + detector: + uses: detection.model@1 + with: + model: construction-ppe-yolov8@1 + model_options: + weights: models/detection/construction-ppe-yolov8/v1/best.pt +``` + +模型加载时会严格比较 checkpoint 的类别名称和顺序与注册信息。权重不匹配时节点会停止 +启动，不能通过只修改 `detect_labels` 绕过类别校验。 + +## 新增版本 + +新增 Detection 模型或模型版本时： + +1. 新建独立的 `/v/` 目录； +2. 放入 Git LFS 管理的权重并编写完整模型卡； +3. 使用 SHA256 校验权重来源和复制结果； +4. 在 `DetectionModelRegistry` 中注册唯一的 `@`； +5. 保证 `supported_labels` 与 checkpoint 类别编号严格同序； +6. 更新部署 YAML，并先执行 `cmvr-edge-ai models` 和 `cmvr-edge-ai validate`。 + +数据集压缩包、训练集和训练过程缓存不属于部署资产，不应放入本目录。 diff --git a/models/detection/construction-ppe-yolov8/v1/README.md b/models/detection/construction-ppe-yolov8/v1/README.md new file mode 100644 index 0000000..6e79b12 --- /dev/null +++ b/models/detection/construction-ppe-yolov8/v1/README.md @@ -0,0 +1,120 @@ +# Construction PPE YOLOv8s v1 + +## 注册信息 + +| 字段 | 值 | +|---|---| +| Model ID | `construction-ppe-yolov8@1` | +| 注册名称 | `Construction PPE YOLOv8s` | +| Backend | `ultralytics-yolo` | +| 任务 | 施工/园区人员 PPE、PPE 缺失违规及部分现场设备检测 | +| 权重 | `models/detection/construction-ppe-yolov8/v1/best.pt` | +| 权重格式 | PyTorch/Ultralytics `.pt` checkpoint | +| SHA256 | `31ef3ca04a17cf545f3fcfc64c4af8993a41d52ccc460e82aff01d5354603533` | + +Model ID 的 `@1` 与本目录的 `v1` 对应。替换权重前必须重新核对 SHA256、checkpoint +标签和评估结果；不兼容的新权重应注册为新版本，不能覆盖本文件记录的 `v1`。 + +## 有序标签 + +checkpoint 的类别编号必须与下表严格一致： + +| ID | 标签 | ID | 标签 | +|---:|---|---:|---| +| 0 | `Boots` | 10 | `No-Helmet` | +| 1 | `Ear-Protection` | 11 | `No-Mask` | +| 2 | `Glass` | 12 | `No-Vest` | +| 3 | `Glove` | 13 | `Worker` | +| 4 | `Hard_hat` | 14 | `Vest` | +| 5 | `Mask` | 15 | `Circular_Saw` | +| 6 | `No-Boots` | 16 | `Fire_Extinguisher` | +| 7 | `No-Ear-Protection` | 17 | `Fire_prevention_Net` | +| 8 | `No-Glass` | 18 | `Welding_Equipment` | +| 9 | `No-Glove` | | | + +`No-*` 是模型直接输出的违规类别，不是框架根据正向 PPE 标签缺失推导出的结果。 + +## 输入与运行参数 + +- 输入必须是已经解码的 packed `BGR8` 或 `RGB8` 图像，buffer 大小为 + `width × height × 3`；adapter 会把 `RGB8` 转换为 backend 使用的 BGR 顺序。 +- 训练/常用推理尺寸为 `640`；实际推理尺寸由 `model_options.imgsz` 控制。 +- 当前项目锁定的 Ultralytics 版本为 `8.4.31`。部署 `.pt` 时还需要与目标设备匹配的 + PyTorch 运行时。 +- CPU 部署使用 `device: cpu`、`half: false`。CUDA/Jetson 必须使用与驱动或 + JetPack 匹配的独立环境，不能只修改 `device`。 + +常用参数： + +| 参数 | 说明 | +|---|---| +| `weights` | 本模型权重路径，必填 | +| `device` | `cpu`、`cuda:0` 或非负 GPU 编号 | +| `imgsz` | 推理尺寸，默认 `640` | +| `iou` | NMS IoU 阈值，默认 `0.70` | +| `half` | 是否使用 FP16，CPU 应保持 `false` | +| `max_det` | 单帧最大检测框数 | +| `agnostic_nms` | 是否启用类别无关 NMS | + +## YAML 示例 + +以下相对路径假定从仓库根目录启动： + +```yaml +detector: + uses: detection.model@1 + with: + model: construction-ppe-yolov8@1 + detect_labels: + - No-Boots + - No-Ear-Protection + - No-Glass + - No-Glove + - No-Helmet + - No-Mask + - No-Vest + confidence: 0.50 + max_fps: 10 + attach_frame: true + model_options: + weights: models/detection/construction-ppe-yolov8/v1/best.pt + device: cpu + imgsz: 640 + iou: 0.70 + half: false + max_det: 100 +``` + +## 训练与评估摘要 + +原模型说明记录了约 13,000 张训练图像、60 个 epoch，以及以下近似结果： + +| 指标 | 原说明记录值 | +|---|---:| +| mAP50 | `~0.76` | +| mAP50-95 | `~0.43` | +| Precision | `~0.81` | +| Recall | `~0.72` | + +这些数据来自原模型说明，未由 cmvr-edge-ai 在目标园区数据上独立复现，不能替代部署前 +的现场验证。 + +## 限制与风险 + +- `Ear-Protection` 样本不足，相关正向或违规结果应谨慎使用。 +- `Boots`、`No-Ear-Protection`、`No-Glove` 等类别可能出现漏检。 +- 远距离小目标、遮挡、运动模糊、低照度、摄像机角度和区域性 PPE 样式会影响效果。 +- 模型尚未证明能泛化到所有园区、工地或人群；告警应用应保留人工复核。 +- 模型同时检测人员、PPE 和现场设备，但不提供目标跟踪，也不保证每件 PPE 与具体人员 + 的空间关联正确。 + +## 许可证与来源 + +- 原模型卡声明权重许可证为 Apache-2.0。 +- 模型基于 Ultralytics YOLOv8；Ultralytics 软件及商业使用可能受其许可证约束，部署方 + 必须自行核对当前适用条款。 +- 训练数据由多个公开来源合并，原说明指出主要来自 Roboflow，并包含其他 GitHub 来源； + 各底层数据集可能有独立许可证。当前模型卡不能替代对训练数据授权链的审查。 +- 权重来源为 Hugging Face 上的 + [`killuminati1/construction-ppe-yolov8`](https://huggingface.co/killuminati1/construction-ppe-yolov8)； + 仓库内以本目录路径和上述 SHA256 作为部署制品标识。 diff --git a/models/detection/construction-ppe-yolov8/v1/best.pt b/models/detection/construction-ppe-yolov8/v1/best.pt new file mode 100644 index 0000000..10297be Binary files /dev/null and b/models/detection/construction-ppe-yolov8/v1/best.pt differ diff --git a/models/detection/ppe-6classes-yolov8n/v1/README.md b/models/detection/ppe-6classes-yolov8n/v1/README.md new file mode 100644 index 0000000..c93b6d4 --- /dev/null +++ b/models/detection/ppe-6classes-yolov8n/v1/README.md @@ -0,0 +1,119 @@ +# PPE Detection YOLOv8n（6 Classes）v1 + +## 注册信息 + +| 字段 | 值 | +|---|---| +| Model ID | `ppe-6classes-yolov8n@1` | +| 注册名称 | `PPE Detection YOLOv8n (6 Classes)` | +| Backend | `ultralytics-yolo` | +| 任务 | 六类 PPE 正向目标检测 | +| 权重 | `models/detection/ppe-6classes-yolov8n/v1/best.pt` | +| 权重格式 | PyTorch/Ultralytics `.pt` checkpoint | +| SHA256 | `07172ef3ae9e256c40a1fb0ce3eefe5547d90170645aa73dded0fffc382cdb31` | + +Model ID 的 `@1` 与本目录的 `v1` 对应。替换权重前必须重新核对 SHA256、checkpoint +标签和评估结果；不兼容的新权重应注册为新版本，不能覆盖本文件记录的 `v1`。 + +## 有序标签 + +checkpoint 的类别编号必须与下表严格一致，大小写也不能改变： + +| ID | 标签 | +|---:|---| +| 0 | `Gloves` | +| 1 | `Vest` | +| 2 | `goggles` | +| 3 | `helmet` | +| 4 | `mask` | +| 5 | `safety_shoe` | + +这六个类别都表示“画面中检测到对应装备”。模型没有 `Person` 或 `No-*` 类，不能把 +“没有检测到 helmet”直接解释为“某个人没有佩戴安全帽”。如需人员违规判断，必须增加 +人员检测、人员与 PPE 空间关联以及缺失判定逻辑。 + +## 输入与运行参数 + +- 输入必须是已经解码的 packed `BGR8` 或 `RGB8` 图像，buffer 大小为 + `width × height × 3`；adapter 会把 `RGB8` 转换为 backend 使用的 BGR 顺序。 +- 训练/常用推理尺寸为 `640`；实际推理尺寸由 `model_options.imgsz` 控制。 +- 当前项目锁定的 Ultralytics 版本为 `8.4.31`。部署 `.pt` 时还需要与目标设备匹配的 + PyTorch 运行时。 +- CPU 部署使用 `device: cpu`、`half: false`。CUDA/Jetson 必须使用与驱动或 + JetPack 匹配的独立环境，不能只修改 `device`。 + +常用参数： + +| 参数 | 说明 | +|---|---| +| `weights` | 本模型权重路径，必填 | +| `device` | `cpu`、`cuda:0` 或非负 GPU 编号 | +| `imgsz` | 推理尺寸，默认 `640` | +| `iou` | NMS IoU 阈值，默认 `0.70` | +| `half` | 是否使用 FP16，CPU 应保持 `false` | +| `max_det` | 单帧最大检测框数 | +| `agnostic_nms` | 是否启用类别无关 NMS | + +## YAML 示例 + +以下相对路径假定从仓库根目录启动： + +```yaml +detector: + uses: detection.model@1 + with: + model: ppe-6classes-yolov8n@1 + detect_labels: + - Gloves + - Vest + - goggles + - helmet + - mask + - safety_shoe + confidence: 0.50 + max_fps: 10 + attach_frame: false + model_options: + weights: models/detection/ppe-6classes-yolov8n/v1/best.pt + device: cpu + imgsz: 640 + iou: 0.70 + half: false + max_det: 100 +``` + +## 训练与评估摘要 + +原模型说明记录了 YOLOv8n 基础模型、50 个 epoch、`imgsz=640`、`batch=32`，以及以下 +近似结果： + +| 指标 | 原说明记录值 | +|---|---:| +| mAP50 | `~0.81` | +| mAP50-95 | `~0.53` | +| Precision | `~0.80` | +| Recall | `~0.74` | + +原说明记录的逐类 mAP50 约为：`Gloves 0.69`、`Vest 0.90`、`goggles 0.90`、 +`helmet 0.90`、`mask 0.80`、`safety_shoe 0.64`。这些数据未由 cmvr-edge-ai 在目标园区 +数据上独立复现，不能替代部署前的现场验证。 + +## 限制与风险 + +- 低照度、运动模糊、远距离小目标、遮挡和不同地区的 PPE 外观会降低准确率。 +- 原数据存在类别不均衡，`goggles` 和 `safety_shoe` 等类别应重点做现场回归测试。 +- 模型只检测 PPE 的出现，不进行人员检测、目标跟踪或人员/PPE 归属判断。 +- 不应在没有人工复核的情况下用于处罚、法律执法或其他高风险决定。 +- 摄像采集和平台上报仍需遵守工作场所隐私、告知和数据保留要求。 + +## 许可证与来源 + +- 原模型卡声明权重许可证为 MIT。 +- 模型基于 Ultralytics YOLOv8；Ultralytics 软件及商业使用可能受其许可证约束，部署方 + 必须自行核对当前适用条款。 +- 原模型说明称训练数据为自定义 Roboflow 格式 PPE 数据集，但没有在本目录提供完整的 + 数据授权链。部署方应在商业或高风险使用前核对数据来源和许可。 +- 本目录仅记录当前 `.pt` 权重，不表示仓库包含其他导出格式。 +- 权重来源为 Hugging Face 上的 + [`Tanishjain9/yolov8n-ppe-detection-6classes`](https://huggingface.co/Tanishjain9/yolov8n-ppe-detection-6classes)； + 仓库内以本目录路径和上述 SHA256 作为部署制品标识。 diff --git a/models/detection/ppe-6classes-yolov8n/v1/best.pt b/models/detection/ppe-6classes-yolov8n/v1/best.pt new file mode 100644 index 0000000..75e3bd3 Binary files /dev/null and b/models/detection/ppe-6classes-yolov8n/v1/best.pt differ diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 960aee7..d996e06 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -9,6 +9,43 @@ cmvr_es_root="${PROJECT_ROOT}/../cmvr-es" skip_codegen=false skip_check=false +verify_model_artifact() { + local model_id="$1" + local relative_path="$2" + local expected_size="$3" + local expected_sha256="$4" + local artifact_path="${PROJECT_ROOT}/${relative_path}" + + if [[ ! -f "${artifact_path}" ]]; then + echo "error: model artifact is missing for ${model_id}: ${relative_path}" >&2 + echo "restore the model file before using the ${profile} profile" >&2 + exit 1 + fi + + if LC_ALL=C grep -q -F "version https://git-lfs.github.com/spec/v1" "${artifact_path}"; then + echo "error: model artifact is only a Git LFS pointer: ${relative_path}" >&2 + echo "run 'git lfs pull' to fetch the real model weights" >&2 + exit 1 + fi + + local actual_size + actual_size="$(wc -c < "${artifact_path}")" + actual_size="${actual_size//[[:space:]]/}" + if [[ "${actual_size}" != "${expected_size}" ]]; then + echo "error: model artifact size mismatch for ${model_id}: ${relative_path}" >&2 + echo "expected ${expected_size} bytes, got ${actual_size} bytes" >&2 + exit 1 + fi + + local actual_sha256 + actual_sha256="$(sha256sum "${artifact_path}" | awk '{print $1}')" + if [[ "${actual_sha256}" != "${expected_sha256}" ]]; then + echo "error: model artifact checksum mismatch for ${model_id}: ${relative_path}" >&2 + echo "expected SHA256 ${expected_sha256}, got ${actual_sha256}" >&2 + exit 1 + fi +} + usage() { cat <<'EOF' Usage: bash scripts/bootstrap.sh [options] @@ -24,7 +61,7 @@ Options: -h, --help show this help Profiles: - core framework and simulated talk/smoke pipeline only + core framework, minimal fixture, and simulated talk pipeline only detection-cpu gRPC + HTTP + PyAV + locked CPU YOLO runtime dev detection-cpu plus tests and portable protobuf codegen tools EOF @@ -138,16 +175,35 @@ if [[ "${skip_check}" == true ]]; then exit 0 fi -echo "==> validating the smoke pipeline" -.venv/bin/cmvr-edge-ai validate --config configs/smoke.yaml +echo "==> validating the minimal framework fixture" +.venv/bin/cmvr-edge-ai validate \ + --config tests/fixtures/minimal_pipeline.yaml \ + --pipeline minimal + +echo "==> validating the merged talk pipeline" +.venv/bin/cmvr-edge-ai validate \ + --config configs/edge_ai.yaml \ + --pipeline talk if [[ "${needs_cmvr_bindings}" == true ]]; then + echo "==> verifying detection model artifacts" + verify_model_artifact \ + "construction-ppe-yolov8@1" \ + "models/detection/construction-ppe-yolov8/v1/best.pt" \ + "22537898" \ + "31ef3ca04a17cf545f3fcfc64c4af8993a41d52ccc460e82aff01d5354603533" + verify_model_artifact \ + "ppe-6classes-yolov8n@1" \ + "models/detection/ppe-6classes-yolov8n/v1/best.pt" \ + "5625014" \ + "07172ef3ae9e256c40a1fb0ce3eefe5547d90170645aa73dded0fffc382cdb31" + echo "==> checking detection runtime imports" .venv/bin/python -c \ "import av, grpc, httpx, PIL, torch, ultralytics; import cmvr.api.camera_service_pb2_grpc; print(f'torch={torch.__version__} cuda={torch.cuda.is_available()} ultralytics={ultralytics.__version__} pillow={PIL.__version__}')" echo "==> validating the PPE detection pipeline" .venv/bin/cmvr-edge-ai validate \ - --config detect_server/pipeline.yaml \ + --config configs/edge_ai.yaml \ --pipeline detection .venv/bin/cmvr-edge-ai models fi diff --git a/src/cmvr_edge_ai/connectors/cmvr_es/__pycache__/camera.cpython-310.pyc b/src/cmvr_edge_ai/connectors/cmvr_es/__pycache__/camera.cpython-310.pyc index 5fee445..432989f 100644 Binary files a/src/cmvr_edge_ai/connectors/cmvr_es/__pycache__/camera.cpython-310.pyc and b/src/cmvr_edge_ai/connectors/cmvr_es/__pycache__/camera.cpython-310.pyc differ diff --git a/src/cmvr_edge_ai/connectors/cmvr_es/camera.py b/src/cmvr_edge_ai/connectors/cmvr_es/camera.py index 7c5c084..0258080 100644 --- a/src/cmvr_edge_ai/connectors/cmvr_es/camera.py +++ b/src/cmvr_edge_ai/connectors/cmvr_es/camera.py @@ -174,6 +174,7 @@ class CmvrCameraRgbStreamSource(Source): self._active_session_id = stream_session_id stats: _StreamStats | None = None progress_task: asyncio.Task[None] | None = None + progress_stop_event: asyncio.Event | None = None async def subscription_requests(): # type: ignore[no-untyped-def] yield self._stream_request @@ -200,8 +201,9 @@ class CmvrCameraRgbStreamSource(Source): reconnect_attempts, ) self._call = self._stub.GetRGBImageStream(subscription_requests()) + progress_stop_event = asyncio.Event() progress_task = asyncio.create_task( - self._log_stream_progress(stats, shutdown_event), + self._log_stream_progress(stats, progress_stop_event), name=f"{self._node_id}:camera-stream-progress", ) received_in_session = False @@ -373,11 +375,13 @@ class CmvrCameraRgbStreamSource(Source): shutdown_event.wait(), timeout=reconnect_delay_s ) return - except TimeoutError: + except (asyncio.TimeoutError, TimeoutError): reconnect_delay_s = min( self._reconnect_max_s, reconnect_delay_s * 2.0 ) finally: + if progress_stop_event is not None: + progress_stop_event.set() if progress_task is not None: progress_task.cancel() await asyncio.gather(progress_task, return_exceptions=True) @@ -516,7 +520,7 @@ class CmvrCameraRgbStreamSource(Source): timeout=self._stream_log_interval_s, ) return - except TimeoutError: + except (asyncio.TimeoutError, TimeoutError): now_ns = monotonic_ns() window_s = max( (now_ns - stats.window_started_ns) / 1_000_000_000, @@ -558,6 +562,10 @@ class CmvrCameraRgbStreamSource(Source): stats.window_bytes = 0 stats.window_key_frames = 0 stats.window_started_ns = now_ns + # Custom Event implementations used by integrations may signal + # a timeout immediately. Always yield after a progress tick so + # logging can never monopolize the application event loop. + await asyncio.sleep(0.001) def _protobuf_timestamp_ns(timestamp: Any) -> int | None: diff --git a/talk_server/README.md b/talk_server/README.md index 2cb2f6b..c57dbe9 100644 --- a/talk_server/README.md +++ b/talk_server/README.md @@ -1,7 +1,22 @@ # Talk pipeline -`pipeline.yaml` is runnable with a simulated audio chunk so the framework can -be tested before the cmvr-es microphone stream proto is available. +The `talk` pipeline in `configs/edge_ai.yaml` is runnable with a simulated audio +chunk so the framework can be tested before the cmvr-es microphone stream proto +is available. It shares the deployment YAML with the `detection` pipeline. + +Run it explicitly from the repository root: + +```bash +uv run --no-sync cmvr-edge-ai validate \ + --config configs/edge_ai.yaml \ + --pipeline talk +uv run --no-sync cmvr-edge-ai run \ + --config configs/edge_ai.yaml \ + --pipeline talk +``` + +Omitting `--pipeline` starts every pipeline whose `enabled` field is `true`, so +use the explicit selector when only the talk process is wanted. When the bidirectional audio RPC lands, replace the synthetic source with the typed `cmvr.grpc.microphone_audio_stream@1` connector and expand the chain to: @@ -12,4 +27,3 @@ AudioChunk -> VAD -> ASR -> dialogue -> TTS -> speaker stream The internal `AudioChunk/v1` contract and port/QoS model are already independent of the final protobuf message names. - diff --git a/talk_server/pipeline.yaml b/talk_server/pipeline.yaml deleted file mode 100644 index 0f65f2f..0000000 --- a/talk_server/pipeline.yaml +++ /dev/null @@ -1,42 +0,0 @@ -api_version: cmvr.edge.ai/v1 - -runtime: - thread_workers: 3 - shutdown_timeout_s: 8 - -pipelines: - talk_smoke: - enabled: true - nodes: - audio_stream_placeholder: - # Replace with cmvr.grpc.microphone_audio_stream@1 when its proto lands. - uses: core.sequence_source@1 - with: - items: - - simulated-audio-chunk - schema_name: AudioChunk - schema_version: 1 - - dialogue_placeholder: - # The real chain will be VAD -> ASR -> dialogue -> TTS. - uses: core.passthrough@1 - - output: - uses: core.log_sink@1 - with: - logger: cmvr_edge_ai.talk - - edges: - - from: audio_stream_placeholder.output - to: dialogue_placeholder.input - qos: - profile: audio_contiguous - capacity: 16 - overflow: block - - - from: dialogue_placeholder.output - to: output.input - qos: - profile: request - capacity: 8 - overflow: block diff --git a/configs/smoke.yaml b/tests/fixtures/minimal_pipeline.yaml similarity index 93% rename from configs/smoke.yaml rename to tests/fixtures/minimal_pipeline.yaml index d148214..09d08ed 100644 --- a/configs/smoke.yaml +++ b/tests/fixtures/minimal_pipeline.yaml @@ -5,7 +5,7 @@ runtime: shutdown_timeout_s: 5 pipelines: - smoke: + minimal: enabled: true nodes: source: @@ -23,7 +23,7 @@ pipelines: sink: uses: core.log_sink@1 with: - logger: cmvr_edge_ai.smoke + logger: cmvr_edge_ai.minimal edges: - from: source.output diff --git a/tests/integration/test_application_cli.py b/tests/integration/test_application_cli.py index 0194d6b..b368344 100644 --- a/tests/integration/test_application_cli.py +++ b/tests/integration/test_application_cli.py @@ -15,17 +15,17 @@ from cmvr_edge_ai.config import load_config PROJECT_ROOT = Path(__file__).resolve().parents[2] -SMOKE_CONFIG = PROJECT_ROOT / "configs" / "smoke.yaml" -DETECTION_CONFIG = PROJECT_ROOT / "detect_server" / "pipeline.yaml" +MINIMAL_CONFIG = PROJECT_ROOT / "tests" / "fixtures" / "minimal_pipeline.yaml" +EDGE_AI_CONFIG = PROJECT_ROOT / "configs" / "edge_ai.yaml" -def test_application_runs_the_finite_smoke_pipeline() -> None: +def test_application_runs_the_finite_minimal_pipeline() -> None: async def exercise(): # type: ignore[no-untyped-def] loop = asyncio.get_running_loop() host_executor = ThreadPoolExecutor(max_workers=1) loop.set_default_executor(host_executor) application = EdgeAIApplication( - load_config(SMOKE_CONFIG), + load_config(MINIMAL_CONFIG), create_default_registry(discover_entry_points=False), ) try: @@ -47,24 +47,35 @@ def test_application_runs_the_finite_smoke_pipeline() -> None: application, running_health, host_executor_result = asyncio.run(exercise()) assert running_health["state"] == "running" - assert set(running_health["pipelines"]) == {"smoke"} - assert application.pipelines == ("smoke",) + assert set(running_health["pipelines"]) == {"minimal"} + assert application.pipelines == ("minimal",) assert application.state is ApplicationState.STOPPED assert host_executor_result == "still-usable" -def test_cli_validates_and_runs_smoke_config( +def test_cli_validates_and_runs_minimal_config( capsys, monkeypatch ) -> None: # type: ignore[no-untyped-def] # ``configure_logging(force=True)`` intentionally owns process logging in # production. Avoid leaking that global CLI side effect into later tests. monkeypatch.setattr("cmvr_edge_ai.cli.configure_logging", lambda *_: None) - assert main(["validate", "--config", str(SMOKE_CONFIG)]) == 0 + assert main(["validate", "--config", str(MINIMAL_CONFIG)]) == 0 validation_output = capsys.readouterr() - assert "configuration is valid; pipelines: smoke" in validation_output.out + assert "configuration is valid; pipelines: minimal" in validation_output.out assert validation_output.err == "" - assert main(["run", "--config", str(SMOKE_CONFIG), "--log-level", "WARNING"]) == 0 + assert ( + main( + [ + "run", + "--config", + str(MINIMAL_CONFIG), + "--log-level", + "WARNING", + ] + ) + == 0 + ) run_output = capsys.readouterr() assert run_output.err == "" @@ -95,16 +106,31 @@ def test_cli_lists_detection_model_metadata(capsys) -> None: # type: ignore[no- assert "Gloves,Vest,goggles,helmet,mask,safety_shoe" in output -def test_real_detection_config_compiles_without_loading_optional_runtimes() -> None: - config = load_config(DETECTION_CONFIG) +def test_merged_config_compiles_each_pipeline_without_loading_optional_runtimes() -> None: + config = load_config(EDGE_AI_CONFIG) + registry = create_default_registry(discover_entry_points=False) + talk = validate_application(config, registry, ("talk",)) compiled = validate_application( config, - create_default_registry(discover_entry_points=False), + registry, + ("detection",), ) + assert set(config.pipelines) == {"detection", "talk"} + assert tuple(item.pipeline_id for item in talk) == ("talk",) assert len(compiled) == 1 + assert compiled[0].pipeline_id == "detection" + assert "ppe_alert_platform" in config.endpoints assert ( - config.pipelines["detection"].nodes["platform"].params["failure_mode"] + config.pipelines["detection"] + .nodes["alert_platform"] + .params["endpoint"] + == "ppe_alert_platform" + ) + assert ( + config.pipelines["detection"] + .nodes["alert_platform"] + .params["failure_mode"] == "log_and_drop" ) assert set(compiled[0].plugin_specs) == { @@ -112,5 +138,5 @@ def test_real_detection_config_compiles_without_loading_optional_runtimes() -> N "decoder", "detector", "repeat_gate", - "platform", + "alert_platform", } diff --git a/tests/integration/test_model_artifacts.py b/tests/integration/test_model_artifacts.py new file mode 100644 index 0000000..58510d7 --- /dev/null +++ b/tests/integration/test_model_artifacts.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path + +import pytest + +from cmvr_edge_ai.config import load_config + + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +CONSTRUCTION_WEIGHTS = "models/detection/construction-ppe-yolov8/v1/best.pt" +MODEL_ARTIFACTS = ( + ( + "construction-ppe-yolov8", + 22_537_898, + "31ef3ca04a17cf545f3fcfc64c4af8993a41d52ccc460e82aff01d5354603533", + ), + ( + "ppe-6classes-yolov8n", + 5_625_014, + "07172ef3ae9e256c40a1fb0ce3eefe5547d90170645aa73dded0fffc382cdb31", + ), +) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as artifact: + for chunk in iter(lambda: artifact.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +@pytest.mark.parametrize( + ("model_name", "expected_size", "expected_sha256"), + MODEL_ARTIFACTS, +) +def test_model_artifact_is_complete( + model_name: str, + expected_size: int, + expected_sha256: str, +) -> None: + model_dir = PROJECT_ROOT / "models" / "detection" / model_name / "v1" + weights = model_dir / "best.pt" + + assert (model_dir / "README.md").is_file() + assert weights.is_file() + assert weights.stat().st_size == expected_size + assert _sha256(weights) == expected_sha256 + + +@pytest.mark.parametrize( + ("config_path", "pipeline_id"), + ( + (PROJECT_ROOT / "configs" / "edge_ai.yaml", "detection"), + ( + PROJECT_ROOT / "configs" / "debug" / "detection_viewer.yaml", + "detection_show", + ), + ), +) +def test_detection_configs_use_repository_construction_weights( + config_path: Path, + pipeline_id: str, +) -> None: + config = load_config(config_path) + detector = config.pipelines[pipeline_id].nodes["detector"] + + assert detector.params["model"] == "construction-ppe-yolov8@1" + assert detector.params["model_options"]["weights"] == CONSTRUCTION_WEIGHTS + assert (PROJECT_ROOT / CONSTRUCTION_WEIGHTS).is_file() diff --git a/tests/unit/test_detection_viewer_demo.py b/tests/unit/test_detection_viewer_demo.py new file mode 100644 index 0000000..8339fc4 --- /dev/null +++ b/tests/unit/test_detection_viewer_demo.py @@ -0,0 +1,506 @@ +from __future__ import annotations + +import asyncio +import importlib.util +from pathlib import Path +import sys +from typing import Any + +import pytest + +from cmvr_edge_ai.application import validate_application +from cmvr_edge_ai.config import load_config +from cmvr_edge_ai.contracts import ( + BoundingBox, + Detection, + DetectionResult, + ImageFrame, +) +from cmvr_edge_ai.core import ComponentContext, Envelope + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +DEMO_CONFIG = PROJECT_ROOT / "configs" / "debug" / "detection_viewer.yaml" +DEMO_SCRIPT = PROJECT_ROOT / "detect_server" / "show_detections.py" + + +def _load_demo_module() -> Any: + spec = importlib.util.spec_from_file_location( + "cmvr_edge_ai_detection_viewer_demo", + DEMO_SCRIPT, + ) + if spec is None or spec.loader is None: + raise RuntimeError(f"could not load demo module: {DEMO_SCRIPT}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +demo = _load_demo_module() + + +class _FakeCv2: + WINDOW_NORMAL = 0 + WND_PROP_VISIBLE = 1 + FONT_HERSHEY_SIMPLEX = 2 + LINE_AA = 3 + COLOR_RGB2BGR = 4 + + def __init__(self, *, key: int = -1, visible: float = 1.0, gui: str = "QT5") -> None: + self.key = key + self.visible = visible + self.gui = gui + self.named: list[tuple[str, int]] = [] + self.resized: list[tuple[str, int, int]] = [] + self.shown: list[tuple[str, Any]] = [] + self.destroyed: list[str] = [] + + def getBuildInformation(self) -> str: + return f"OpenCV test build\n GUI: {self.gui}\n" + + def namedWindow(self, name: str, mode: int) -> None: + self.named.append((name, mode)) + + def resizeWindow(self, name: str, width: int, height: int) -> None: + self.resized.append((name, width, height)) + + def imshow(self, name: str, image: Any) -> None: + self.shown.append((name, image)) + + def waitKey(self, delay: int) -> int: + del delay + return self.key + + def getWindowProperty(self, name: str, prop: int) -> float: + del name, prop + return self.visible + + def destroyWindow(self, name: str) -> None: + self.destroyed.append(name) + + +def _frame( + *, + pixel_format: str = "BGR8", + codec: str = "none", + data: bytes | None = None, +) -> ImageFrame: + return ImageFrame( + data=bytes((10, 20, 30)) * 4 if data is None else data, + width=2, + height=2, + pixel_format=pixel_format, + codec=codec, + ) + + +def _result(*, source_frame: ImageFrame | None = None) -> DetectionResult: + return DetectionResult( + detections=( + Detection( + "No-Helmet", + 0.91, + BoundingBox(0.0, 0.0, 1.0, 1.0), + ), + ), + model_id="construction-ppe-yolov8@1", + model_name="Construction PPE YOLOv8s", + inference_ms=12.5, + source_frame=_frame() if source_frame is None else source_frame, + ) + + +def _envelope(payload: Any) -> Envelope[Any]: + return Envelope( + payload, + schema_name="DetectionResult", + schema_version=1, + source_id="wrist_cam", + sequence=7, + ) + + +def _context() -> ComponentContext: + return ComponentContext( + pipeline_id="detection_show", + node_id="viewer", + shutdown_event=asyncio.Event(), + metadata={}, + ) + + +def test_demo_config_compiles_with_only_camera_decoder_detector_and_viewer() -> None: + config = load_config(DEMO_CONFIG) + compiled = validate_application( + config, + demo._build_registry(lambda: None), + ("detection_show",), + ) + + assert len(compiled) == 1 + assert set(compiled[0].plugin_specs) == { + "camera", + "decoder", + "detector", + "viewer", + } + detector = config.pipelines["detection_show"].nodes["detector"] + assert detector.params["max_fps"] == 10 + assert detector.params["attach_frame"] is True + viewer_edge = next( + edge + for edge in config.pipelines["detection_show"].edges + if edge.target == "viewer.input" + ) + assert viewer_edge.source == "detector.detections" + assert viewer_edge.qos.profile == "realtime_latest" + assert viewer_edge.qos.capacity == 1 + assert viewer_edge.qos.overflow == "drop_oldest" + + +@pytest.mark.parametrize( + ("params", "message"), + [ + ({"unknown": 1}, "unknown detection viewer parameter"), + ({"window_name": ""}, "window_name must be a non-empty"), + ({"window_width": True}, "window_width must be an integer"), + ({"window_height": 0}, "window_height must be between"), + ({"wait_key_ms": 0}, "wait_key_ms must be between"), + ({"wait_key_ms": 21}, "wait_key_ms must be between"), + ({"box_thickness": 21}, "box_thickness must be between"), + ({"font_scale": float("nan")}, "font_scale must be between"), + ({"show_stats": "true"}, "show_stats must be a boolean"), + ], +) +def test_viewer_parameters_are_strict(params: dict[str, Any], message: str) -> None: + with pytest.raises(ValueError, match=message): + demo.OpenCvDetectionViewerSink( + "viewer", + params, + request_stop=lambda: None, + ) + + +def test_box_clipping_skips_invalid_or_fully_outside_boxes() -> None: + valid = Detection("valid", 0.9, BoundingBox(-2.2, 1.2, 20.0, 9.8)) + reversed_box = Detection("bad", 0.9, BoundingBox(8.0, 8.0, 2.0, 2.0)) + outside = Detection("outside", 0.9, BoundingBox(12.0, 2.0, 20.0, 5.0)) + not_finite = Detection("nan", 0.9, BoundingBox(float("nan"), 1, 2, 3)) + + assert demo._clipped_box(valid, 10, 8) == (0, 1, 9, 7) + assert demo._clipped_box(reversed_box, 10, 8) is None + assert demo._clipped_box(outside, 10, 8) is None + assert demo._clipped_box(not_finite, 10, 8) is None + + +@pytest.mark.parametrize( + ("frame", "message"), + [ + (_frame(codec="H265"), "requires a decoded"), + (_frame(pixel_format="GRAY8"), "supports BGR8 or RGB8"), + (_frame(data=b"too-short"), "packed frame has"), + ( + ImageFrame(b"", 0, 2, "BGR8"), + "dimensions must be positive integers", + ), + ( + ImageFrame(b"", 2, 2, "BGR8", codec=None), # type: ignore[arg-type] + "codec must be a string", + ), + ], +) +def test_display_frame_validation_is_actionable( + frame: ImageFrame, + message: str, +) -> None: + with pytest.raises(demo.DetectionViewerError, match=message): + demo._validate_display_frame(frame) + + +def test_viewer_rejects_missing_source_frame( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_cv2 = _FakeCv2() + stop_requested = False + + def request_stop() -> None: + nonlocal stop_requested + stop_requested = True + + def load_module(name: str) -> Any: + return fake_cv2 if name == "cv2" else object() + + async def scenario() -> None: + viewer = demo.OpenCvDetectionViewerSink( + "viewer", + {}, + request_stop=request_stop, + module_loader=load_module, + ) + await viewer.setup(_context()) + result = DetectionResult((), "model@1", 1.0, source_frame=None) + with pytest.raises(demo.DetectionViewerError, match="attach_frame: true"): + await viewer.consume(_envelope(result)) + await viewer.stop() + + monkeypatch.setenv("DISPLAY", ":99") + asyncio.run(scenario()) + assert stop_requested is False + + +@pytest.mark.parametrize("key", [ord("q"), ord("Q"), 27]) +def test_viewer_quit_keys_work_before_any_detection_frame( + key: int, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_cv2 = _FakeCv2(key=key) + stop_calls = 0 + stop_event: asyncio.Event | None = None + + def request_stop() -> None: + nonlocal stop_calls, stop_event + stop_calls += 1 + assert stop_event is not None + stop_event.set() + + def load_module(name: str) -> Any: + return fake_cv2 if name == "cv2" else object() + + monkeypatch.setenv("DISPLAY", ":99") + + async def scenario() -> None: + nonlocal stop_event + stop_event = asyncio.Event() + viewer = demo.OpenCvDetectionViewerSink( + "viewer", + {}, + request_stop=request_stop, + module_loader=load_module, + ) + await viewer.setup(_context()) + await viewer.start() + await asyncio.wait_for(stop_event.wait(), timeout=0.2) + await viewer.stop() + await viewer.stop() + + asyncio.run(scenario()) + assert stop_calls == 1 + assert fake_cv2.shown == [] + assert fake_cv2.destroyed == ["CMVR PPE Detection"] + + +def test_viewer_window_close_requests_outer_application_stop( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_cv2 = _FakeCv2(visible=0.0) + stop_calls = 0 + stop_event: asyncio.Event | None = None + + def request_stop() -> None: + nonlocal stop_calls, stop_event + stop_calls += 1 + assert stop_event is not None + stop_event.set() + + monkeypatch.setenv("DISPLAY", ":99") + + async def scenario() -> None: + nonlocal stop_event + stop_event = asyncio.Event() + viewer = demo.OpenCvDetectionViewerSink( + "viewer", + {}, + request_stop=request_stop, + module_loader=lambda name: fake_cv2 if name == "cv2" else object(), + ) + await viewer.setup(_context()) + await viewer.start() + await asyncio.wait_for(stop_event.wait(), timeout=0.2) + await viewer.stop() + + asyncio.run(scenario()) + assert stop_calls == 1 + + +def test_unknown_visibility_does_not_close_window( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_cv2 = _FakeCv2(visible=-1.0) + stop_calls = 0 + + def request_stop() -> None: + nonlocal stop_calls + stop_calls += 1 + + monkeypatch.setenv("DISPLAY", ":99") + + async def scenario() -> None: + viewer = demo.OpenCvDetectionViewerSink( + "viewer", + {}, + request_stop=request_stop, + module_loader=lambda name: fake_cv2 if name == "cv2" else object(), + ) + await viewer.setup(_context()) + await viewer.start() + await asyncio.sleep(0.03) + await viewer.stop() + + asyncio.run(scenario()) + assert stop_calls == 0 + + +def test_consume_displays_rendered_detection_frame( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_cv2 = _FakeCv2() + monkeypatch.setattr( + demo, + "_render_detection_frame", + lambda *args, **kwargs: "frame", + ) + monkeypatch.setenv("DISPLAY", ":99") + + async def scenario() -> None: + viewer = demo.OpenCvDetectionViewerSink( + "viewer", + {}, + request_stop=lambda: None, + module_loader=lambda name: fake_cv2 if name == "cv2" else object(), + ) + await viewer.setup(_context()) + await viewer.start() + await viewer.consume(_envelope(_result())) + await viewer.stop() + + asyncio.run(scenario()) + assert fake_cv2.shown == [("CMVR PPE Detection", "frame")] + + +def test_linux_headless_session_fails_before_loading_opencv( + monkeypatch: pytest.MonkeyPatch, +) -> None: + modules_loaded: list[str] = [] + viewer = demo.OpenCvDetectionViewerSink( + "viewer", + {}, + request_stop=lambda: None, + module_loader=lambda name: modules_loaded.append(name), + ) + monkeypatch.setattr(demo.sys, "platform", "linux") + monkeypatch.delenv("DISPLAY", raising=False) + monkeypatch.delenv("WAYLAND_DISPLAY", raising=False) + + with pytest.raises(demo.DetectionViewerError, match="DISPLAY"): + asyncio.run(viewer.setup(_context())) + assert modules_loaded == [] + + +def test_headless_opencv_build_is_rejected( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_cv2 = _FakeCv2(gui="NONE") + viewer = demo.OpenCvDetectionViewerSink( + "viewer", + {}, + request_stop=lambda: None, + module_loader=lambda name: fake_cv2 if name == "cv2" else object(), + ) + monkeypatch.setenv("DISPLAY", ":99") + + with pytest.raises(demo.DetectionViewerError, match="no GUI backend"): + asyncio.run(viewer.setup(_context())) + assert fake_cv2.named == [] + + +def test_render_preserves_bgr_and_converts_rgb() -> None: + numpy = pytest.importorskip("numpy") + + class DrawingCv2(_FakeCv2): + def __init__(self) -> None: + super().__init__() + self.operations: list[tuple[Any, ...]] = [] + + def cvtColor(self, image: Any, code: int) -> Any: + assert code == self.COLOR_RGB2BGR + return image[:, :, ::-1].copy() + + def rectangle(self, *args: Any, **kwargs: Any) -> None: + del kwargs + _, top_left, bottom_right, _, thickness = args + self.operations.append( + ("rectangle", top_left, bottom_right, thickness) + ) + + def getTextSize(self, *args: Any, **kwargs: Any) -> tuple[tuple[int, int], int]: + del args, kwargs + return (10, 5), 1 + + def putText(self, *args: Any, **kwargs: Any) -> None: + del kwargs + self.operations.append(("text", args[1])) + + cv2 = DrawingCv2() + bgr = _frame(pixel_format="BGR8") + rgb = _frame(pixel_format="RGB8") + kwargs = { + "cv2_module": cv2, + "numpy_module": numpy, + "box_thickness": 2, + "font_scale": 0.6, + } + + bgr_image = demo._render_detection_frame( + bgr, + _result().detections, + header="stats", + **kwargs, + ) + rgb_image = demo._render_detection_frame(rgb, (), header=None, **kwargs) + + assert tuple(int(value) for value in bgr_image[0, 0]) == (10, 20, 30) + assert tuple(int(value) for value in rgb_image[0, 0]) == (30, 20, 10) + assert bgr.data == bytes((10, 20, 30)) * 4 + assert cv2.operations[:2] == [ + ("rectangle", (0, 0), (1, 1), -1), + ("text", "stats"), + ] + assert ("rectangle", (0, 0), (1, 1), 2) in cv2.operations[2:] + assert ("text", "No-Helmet 0.91") in cv2.operations[2:] + + +def test_run_demo_viewer_callback_stops_outer_application( + monkeypatch: pytest.MonkeyPatch, +) -> None: + applications: list[Any] = [] + + class FakeApplication: + def __init__(self, config: Any, registry: Any) -> None: + del config + self.registry = registry + self.started_with: tuple[str, ...] | None = None + self.stop_calls = 0 + applications.append(self) + + async def start(self, pipeline_ids: tuple[str, ...]) -> None: + self.started_with = pipeline_ids + viewer = self.registry.resolve(demo._VIEWER_PLUGIN_ID).factory( + "viewer", + {}, + ) + viewer._request_demo_stop("test") + + async def wait(self) -> None: + await asyncio.Event().wait() + + async def stop(self, *, graceful: bool) -> None: + assert graceful is True + self.stop_calls += 1 + + monkeypatch.setattr(demo, "EdgeAIApplication", FakeApplication) + config = load_config(DEMO_CONFIG) + + assert asyncio.run(demo._run_demo(config, "detection_show")) == 0 + assert len(applications) == 1 + assert applications[0].started_with == ("detection_show",) + assert applications[0].stop_calls >= 1