diff --git a/CMakeLists.txt b/CMakeLists.txt index 0609aed3..3d0ebabb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,6 +2,8 @@ cmake_minimum_required(VERSION 3.22.0) project(cmvr_es) +include(CTest) + # 设置C++标准 set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -18,6 +20,34 @@ set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/output" CACHE PATH "" FORCE) set(CMAKE_BUILD_RPATH "\$ORIGIN:\$ORIGIN/../lib") set(CMAKE_INSTALL_RPATH "\$ORIGIN:\$ORIGIN/../lib") +# Some vendor SDK directories contain an obsolete private libstdc++.so.6. They +# are still needed in the build RUNPATH for their own shared objects, but must +# not shadow the C++ runtime selected by the active compiler. +execute_process( + COMMAND ${CMAKE_CXX_COMPILER} -print-file-name=libstdc++.so.6 + OUTPUT_VARIABLE CMVR_COMPILER_LIBSTDCXX + OUTPUT_STRIP_TRAILING_WHITESPACE +) +if(IS_ABSOLUTE "${CMVR_COMPILER_LIBSTDCXX}" AND EXISTS "${CMVR_COMPILER_LIBSTDCXX}") + # Do not put the whole system library directory first: that can mix the + # system FFmpeg with the repository's bundled FFmpeg. A one-library shim + # fixes only the vendor libstdc++ collision. + set(CMVR_COMPILER_RUNTIME_DIR "${CMAKE_BINARY_DIR}/cmvr_compiler_runtime") + file(MAKE_DIRECTORY "${CMVR_COMPILER_RUNTIME_DIR}") + file(CREATE_LINK + "${CMVR_COMPILER_LIBSTDCXX}" + "${CMVR_COMPILER_RUNTIME_DIR}/libstdc++.so.6" + SYMBOLIC COPY_ON_ERROR + RESULT CMVR_COMPILER_RUNTIME_LINK_RESULT) + if(CMVR_COMPILER_RUNTIME_LINK_RESULT STREQUAL "0") + list(PREPEND CMAKE_BUILD_RPATH "${CMVR_COMPILER_RUNTIME_DIR}") + else() + message(WARNING + "Could not prepare compiler libstdc++ build shim: " + "${CMVR_COMPILER_RUNTIME_LINK_RESULT}") + endif() +endif() + # Use RUNPATH (new dtags) generally preferable set(CMAKE_BUILD_WITH_INSTALL_RPATH OFF) set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) @@ -27,7 +57,20 @@ set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) ############################################################# list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") include(FindExternalLib) -set(ARCH "x86") +set(CMVR_ARCH "" CACHE STRING "Dependency architecture directory (x86 or arm)") +if(NOT CMVR_ARCH) + string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _cmvr_system_processor) + if(_cmvr_system_processor MATCHES "^(aarch64|arm64|arm)") + set(CMVR_ARCH "arm") + else() + set(CMVR_ARCH "x86") + endif() +endif() +set_property(CACHE CMVR_ARCH PROPERTY STRINGS x86 arm) +if(NOT CMVR_ARCH STREQUAL "x86" AND NOT CMVR_ARCH STREQUAL "arm") + message(FATAL_ERROR "CMVR_ARCH must be either 'x86' or 'arm', got: ${CMVR_ARCH}") +endif() +set(ARCH "${CMVR_ARCH}") setup_external_libs(${ARCH}) # 在调用 setup_external_libs 之后 message(STATUS "CMAKE_EXE_LINKER_FLAGS: ${CMAKE_EXE_LINKER_FLAGS}") @@ -35,6 +78,20 @@ message(STATUS "CMAKE_SHARED_LINKER_FLAGS: ${CMAKE_SHARED_LINKER_FLAGS}") find_package(protobuf REQUIRED) find_package(gRPC REQUIRED) +option(CMVR_ENABLE_MSQUIC_BACKEND + "Enable the MsQuic backend for the QUIC edge service when available" + ON) +set(CMVR_HAS_MSQUIC OFF) +if(CMVR_ENABLE_MSQUIC_BACKEND) + find_package(MsQuic QUIET) + if(MsQuic_FOUND) + set(CMVR_HAS_MSQUIC ON) + message(STATUS "MsQuic found: ${MsQuic_LIBRARY}") + else() + message(STATUS "MsQuic not found; building QUIC edge in unavailable/stub mode") + endif() +endif() + ############################################################ # PROTO , 遇到proto 的生成问题，现在build 目录执行 cmake --install . --verbose ，然后编译即可 ############################################################ @@ -109,7 +166,7 @@ target_include_directories(cmvr_es PRIVATE ${GLOG_INCLUDE_DIRS}) target_link_libraries(cmvr_es PRIVATE cmvr_es::proto cmvr_es::logging - service + cmvr_es::quic_edge_task ${GLOG_LIBRARIES} jsoncpp cmvr_es::service diff --git a/README.md b/README.md index 03b74a5e..ec1a4999 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,1085 @@ # CMVR-ES -## Overview +CMVR-ES（CMVR Edge System）是部署在机器人边缘主机上的 C++17 运行时。它负责统一加载设备与任务、通过 gRPC 提供稳定的设备控制接口，并通过 QUIC 主动连接平台，上报节点在线状态、IP 地址以及允许丢帧的实时音视频。 -## Installation +本项目的协议边界是： -### 1. Git submodules install +- 机械臂、AGV、灵巧手等控制与状态查询继续使用 gRPC； +- 节点注册、心跳和 IP 主动上报使用 QUIC 可靠流； +- 摄像头和麦克风实时数据使用 QUIC DATAGRAM； +- 浏览器不直接连接 CMVR 自定义 QUIC 协议，而是连接平台侧网关提供的 WebTransport、WebRTC、MSE/WebSocket 或其他 Web 接口。 -``` -git submodule update --init --recursive +## 当前状态 + +| 项目 | 当前状态 | +| --- | --- | +| 已验证构建平台 | Ubuntu 22.04 x86_64、GCC 11.4、CMake 3.30.3 | +| CMake / C++ 最低要求 | CMake 3.22、C++17 | +| ARM | 目录和部分依赖已准备，但当前依赖树不完整，尚未形成可复现的完整构建 | +| gRPC | 默认启用，监听 `0.0.0.0:50052`，reflection 默认开启 | +| QUIC | 默认关闭；找到 MsQuic 时构建真实后端，否则构建不可用占位后端 | +| 物理设备 | 默认全部关闭，适合没有连接设备的开发主机 | +| 平台网关 / Web 前端 | 不在本仓库中，需要平台项目按本文协议接入 | + +> [!IMPORTANT] +> 当前可复现构建目标是 `x86`。请不要把 `arm` 目录存在理解为 ARM 全量构建已经通过。 + +## 目录 + +- [1. 工程概览](#1-工程概览) +- [2. 系统架构](#2-系统架构) +- [3. 目录结构](#3-目录结构) +- [4. 依赖与环境准备](#4-依赖与环境准备) +- [5. 构建与安装](#5-构建与安装) +- [6. 配置说明](#6-配置说明) +- [7. 运行与测试](#7-运行与测试) +- [8. 生产部署](#8-生产部署) +- [9. 平台端接入](#9-平台端接入) +- [10. 开发扩展指南](#10-开发扩展指南) +- [11. 安全说明](#11-安全说明) +- [12. 已知限制与排障](#12-已知限制与排障) + +## 1. 工程概览 + +### 1.1 核心能力 + +| 能力 | 说明 | +| --- | --- | +| 设备统一管理 | 通过设备抽象、设备工厂和 `DeviceManager` 加载不同厂商实现 | +| 任务统一管理 | 通过 `TaskManager` 运行周期任务和阻塞式服务任务 | +| gRPC 控制面 | 提供系统、相机、音频、机械臂、AGV、灵巧手和生物头等服务 | +| QUIC 节点在线面 | 边缘端主动连接平台，完成注册、心跳、IP 和 gRPC 地址上报 | +| QUIC 实时媒体面 | 通过 QUIC DATAGRAM 发送可丢弃的音视频帧，控制信息仍走可靠流 | +| 协议无关媒体分发 | `MediaSourceHub` 已供 gRPC RGB/麦克风流和 QUIC 彩色/麦克风轨道共享采集源 | +| 算法与仿真 | 包含运动学、轨迹规划、控制算法以及部分 MuJoCo 接入 | + +### 1.2 代码中已包含的设备实现 + +下表表示仓库中存在对应代码，并不表示所有型号都已在当前主机和当前版本完成真机验证。 + +| 类别 | 已包含实现 | +| --- | --- | +| 摄像头 | UVC、RealSense、Hikvision、Mech-Eye、MuJoCo Camera | +| 麦克风 / 音箱 | FFmpeg Microphone、FFmpeg Speaker | +| 机械臂 | AUBO、Huayan、Motor Robot Arm | +| AGV | SRC1100、My AGV | +| 灵巧手 | PX-6AX Gen3、RH56DFTP | +| 生物头 | ESP32 BioHead | +| 电机与总线 | TI5 Motor、MuJoCo Motor、CAN / CANopen、PCAN | + +### 1.3 协议职责 + +| 数据 | 推荐通道 | 原因 | +| --- | --- | --- | +| 机械臂、AGV 控制命令 | gRPC | 需要明确的请求结果、错误处理和稳定语义 | +| 设备状态和管理接口 | gRPC | 便于平台生成强类型客户端并统一治理 | +| 节点注册、心跳、IP 上报 | QUIC reliable stream | 边缘端主动连接，复用 QUIC TLS 会话和重连机制 | +| 视频、音频实时帧 | QUIC DATAGRAM | 网络拥塞时允许丢弃过期帧，避免旧数据阻塞实时数据 | +| 浏览器控制和播放 | 平台 Web API | 浏览器不能直接使用本项目的自定义 QUIC ALPN | + +## 2. 系统架构 + +```mermaid +flowchart LR + MAIN["cmvr_es / main.cpp"] --> CONFIG["Proto Text 配置"] + MAIN --> DEVICE_MANAGER["DeviceManager"] + MAIN --> TASK_MANAGER["TaskManager"] + + DEVICE_MANAGER --> DEVICE_FACTORY["DeviceFactory"] + DEVICE_FACTORY --> DEVICES["设备抽象与厂商驱动"] + DEVICES --> HARDWARE["串口 / CAN / 厂商 SDK / 仿真"] + + TASK_MANAGER --> GRPC_TASK["GrpcServerTask"] + TASK_MANAGER --> QUIC_TASK["QuicEdgeTask"] + TASK_MANAGER --> PERIODIC_TASK["周期控制任务"] + + GRPC_TASK --> GRPC_SERVICE["gRPC Services"] + GRPC_SERVICE --> DEVICE_MANAGER + GRPC_SERVICE -->|"RGB / Microphone stream"| MEDIA_HUB["MediaSourceHub"] + + QUIC_TASK --> QUIC_SERVICE["QuicEdgeService"] + QUIC_SERVICE --> MEDIA_HUB + MEDIA_HUB --> MEDIA_ADAPTER["DeviceMediaSourceAdapter"] + MEDIA_ADAPTER --> DEVICES + + QUIC_SERVICE -->|"可靠流：注册 / 心跳 / 媒体描述"| GATEWAY["Java / 平台 QUIC Gateway"] + QUIC_SERVICE -->|"DATAGRAM：音视频"| GATEWAY + GATEWAY -->|"WebTransport / WebRTC / HTTP"| BROWSER["Browser"] + GATEWAY -->|"根据节点上报地址调用"| GRPC_SERVICE ``` -### 2. Dependency install +### 2.1 启动流程 -```shell -# basic +1. 程序从 `/proc/self/exe` 获取可执行文件目录。 +2. 无参数启动时加载可执行文件旁的 `config/cmvr_es.pb.txt`；传入一个参数时加载指定根配置。 +3. 以根配置文件所在目录作为其他业务配置和证书相对路径的配置根目录。 +4. 加载日志配置并初始化日志。 +5. `DeviceManager` 根据配置创建并初始化启用的设备。 +6. 注册 gRPC 和 QUIC 任务工厂。 +7. `TaskManager` 创建并启动启用的周期任务或阻塞式服务任务。 +8. 主线程等待 `SIGINT` 或 `SIGTERM`。 +9. 收到退出信号后停止任务并关闭日志。 + +当前主退出路径会停止 `TaskManager` 中的任务，但没有显式调用 `DeviceManager::stop()`。需要统一关闭全部设备时，可调用 `SystemService/StopAll`；后续也应考虑补齐进程退出时的设备级统一停止。 + +### 2.2 MediaSourceHub + +`MediaSourceHub` 是设备层与传输协议之间的协议无关媒体层。当前 gRPC 相机 RGB 流、gRPC 麦克风流，以及 QUIC 相机彩色轨道和麦克风轨道使用该 Hub；gRPC Depth/RGBD 流仍直接读取设备编码帧，尚未迁移到 Hub。 + +- 已接入 Hub 的 gRPC 与 QUIC 流共享同一设备采集源，后续新增协议也应从 Hub 订阅； +- 第一个订阅者启动设备流，最后一个订阅者释放设备流； +- 每个消费线程必须使用独立的 `Subscription`，读取游标不能跨线程共享； +- 描述信息、帧及其 payload 均为不可变对象，可以安全地跨协议共享； +- 环形队列有界，慢消费者不会阻塞生产者； +- 被覆盖的帧会形成精确的 dropped count，协议层据此标记 discontinuity； +- H.264/H.265 出现丢帧或 generation 变化后，协议层可通过 Hub 请求新的关键帧； +- source `start` 必须响应 cancellation，source `stop` 必须同步解除回调并回收生产线程。 + +关键文件： + +- [`media_source_hub.h`](cmvr-es/manager/media_source_hub/include/media_source_hub.h) +- [`device_media_source_adapter.cpp`](cmvr-es/manager/media_source_hub/src/device_media_source_adapter.cpp) +- [`media_frame.h`](cmvr-es/common/media/media_frame.h) +- [`ring_buffer.h`](cmvr-es/common/base/ring_buffer.h) + +## 3. 目录结构 + +```text +cmvr-es/ +├── CMakeLists.txt +├── README.md +├── LICENSE +├── request.txt # 三方依赖清单 +├── cmake/ +│ ├── FindExternalLib.cmake # 仓库内依赖发现与安装 +│ └── FindMsQuic.cmake # 可选 MsQuic 后端发现 +├── protos/ +│ └── cmvr/ +│ ├── api/ # 平台 gRPC API +│ ├── common/ # 通用协议类型 +│ ├── config/ # Proto Text 配置 Schema +│ ├── msgs/ # 设备和状态消息 +│ └── quic_edge/v1/ # QUIC 控制面、媒体协议与线协议文档 +├── cmvr-es/ +│ ├── main.cpp # 进程入口与生命周期 +│ ├── config/ # 默认运行配置 +│ ├── common/ # 日志、配置、基础容器和媒体模型 +│ ├── hardware/ # 串口等底层能力 +│ ├── devices/ # 设备抽象、工厂和厂商实现 +│ ├── manager/ +│ │ ├── device_manager/ +│ │ ├── task_manager/ +│ │ └── media_source_hub/ +│ ├── service/ +│ │ ├── grpc/ +│ │ └── quic_edge/ +│ ├── task/ +│ │ ├── grpc_server_task/ +│ │ ├── quic_edge_task/ +│ │ └── touch_screen_task/ +│ ├── algorithms/ +│ └── simulate/ +├── dependency/ +│ ├── x86/third_party/ # x86_64 预置依赖 +│ └── arm/third_party/ # ARM 部分依赖，当前尚未完整验证 +├── model/ # 运行模型和机器人资源 +├── assets/ # SDK 安装包、grpcurl 等辅助资产 +├── python/ # 标定和辅助脚本 +├── script/ # 历史辅助脚本 +├── build/ # 本机构建目录，不纳入 Git +└── output/ # cmake --install 产物，不纳入 Git +``` + +### 3.1 模块开发文档 + +核心目录提供独立的扩展指南。新增模块时应先阅读目标目录文档，再沿其中列出的 Proto、配置、工厂、CMake 和测试链路完成接入。 + +| 模块 | 开发文档 | 主要内容 | +| --- | --- | --- | +| 设备层 | [`cmvr-es/devices/README.md`](cmvr-es/devices/README.md) | 新设备后端、新设备类别、生命周期和实时流安全 | +| 管理层 | [`cmvr-es/manager/README.md`](cmvr-es/manager/README.md) | DeviceManager、TaskManager、MediaSourceHub | +| 服务层 | [`cmvr-es/service/README.md`](cmvr-es/service/README.md) | gRPC Service 注册和 QUIC 协议引擎扩展 | +| 任务层 | [`cmvr-es/task/README.md`](cmvr-es/task/README.md) | Task 接口、运行模式、Factory 和配置接入 | +| 公共层 | [`cmvr-es/common/README.md`](cmvr-es/common/README.md) | 公共类型、媒体模型、环形队列和配置工具 | +| 算法层 | [`cmvr-es/algorithms/README.md`](cmvr-es/algorithms/README.md) | IK、规划、控制、感知和算法工厂 | +| 硬件层 | [`cmvr-es/hardware/README.md`](cmvr-es/hardware/README.md) | 串口/总线抽象、并发事务和权限 | +| 配置层 | [`cmvr-es/config/README.md`](cmvr-es/config/README.md) | Proto Text 配置树、ID 和生产配置 | +| 仿真层 | [`cmvr-es/simulate/README.md`](cmvr-es/simulate/README.md) | MuJoCo、仿真引擎和模拟设备边界 | +| 协议定义 | [`protos/README.md`](protos/README.md) | Proto 生成、兼容规则、Java 和 QUIC 版本 | + +## 4. 依赖与环境准备 + +### 4.1 基础环境 + +已验证环境： + +- Ubuntu 22.04.5 LTS x86_64； +- GCC 11.4； +- CMake 3.30.3； +- C++17； +- 建议至少预留 10 GB 可用磁盘空间用于依赖、构建和安装产物。 + +工程要求 CMake 3.22 或更高版本。若系统 CMake 版本过低，可使用 [`assets/`](assets/) 中随仓库提供的对应架构 CMake 安装包，或由系统管理员安装新版 CMake。 + +### 4.2 系统依赖 + +下面的命令面向 Ubuntu 22.04。基础构建依赖： + +```bash sudo apt-get update -sudo apt install -y build-essential cmake git pkg-config patchelf libboost-all-dev libssl-dev - -# opencv -sudo apt install -y \ - libjpeg-dev libpng-dev libtiff-dev \ - libavcodec-dev libavformat-dev libswscale-dev \ - libv4l-dev libxvidcore-dev libx264-dev \ - libgtk-3-dev - -# PulseAudio -sudo apt-get install libpulse-dev - -# OpenGL -sudo apt-get install libgl1-mesa-dev - -# GLFW -sudo apt-get install libglfw3-dev - -# Assimp -sudo apt-get install libassimp-dev - -# visp -sudo apt-get install -y libx11-dev liblapack-dev libzbar-dev libpthread-stubs0-dev libdc1394-dev nlohmann-json3-dev - -# realsense sudo apt-get install -y \ - libusb-1.0-0-dev libudev-dev \ - libglu1-mesa-dev - -# Matplot++ -sudo apt install gnuplot-qt + build-essential \ + cmake \ + git \ + pkg-config \ + patchelf \ + file \ + libboost-all-dev \ + libssl-dev ``` +图像、仿真和设备接入常用依赖： +```bash +sudo apt-get install -y \ + libjpeg-dev \ + libpng-dev \ + libtiff-dev \ + libv4l-dev \ + libgtk-3-dev \ + libgl1-mesa-dev \ + libglu1-mesa-dev \ + libglfw3-dev \ + libassimp-dev \ + libx11-dev \ + liblapack-dev \ + libblas-dev \ + libzbar-dev \ + libpthread-stubs0-dev \ + libdc1394-dev \ + nlohmann-json3-dev \ + libxml2-dev \ + libpulse-dev \ + libasound2-dev \ + libusb-1.0-0-dev \ + libudev-dev +``` + +按实际设备选择安装： + +```bash +# AUBO SDK 的 Qt 相关能力 +sudo apt-get install -y qtbase5-dev + +# CAN 调试工具 +sudo apt-get install -y can-utils + +# Matplot++ 交互绘图 +sudo apt-get install -y gnuplot-qt +``` + +FFmpeg、OpenCV、gRPC、Protobuf、RealSense、Hikvision SDK、AUBO SDK、Pinocchio、MuJoCo、ViSP、OSQP 等主要 C/C++ 依赖优先从 `dependency//third_party/` 查找。完整清单和版本以 [`request.txt`](request.txt) 与 [`FindExternalLib.cmake`](cmake/FindExternalLib.cmake) 为准。 + +### 4.3 仓库依赖注意事项 + +当前依赖管理方式有两个需要特别注意的历史问题： + +1. 三方依赖的主要来源是仓库中的 `dependency//third_party/`，不是 Git submodule。 +2. 当前 `.gitmodules` 与仓库实际 gitlink 不一致，执行 `git submodule update --init --recursive` 会因 `assets/toppra` 缺少映射而失败。 + +因此，不要把旧版 README 中的 submodule 命令作为安装步骤。`script/install.sh` 也包含旧目录和硬编码架构，当前不作为标准安装入口。 + +## 5. 构建与安装 + +以下命令都从仓库根目录执行。 + +### 5.1 首次构建工具引导 + +根 CMake 当前固定从 `output/bin/protoc` 和 `output/bin/grpc_cpp_plugin` 生成 C++ protobuf/gRPC 代码，而 `output/` 不纳入 Git。因此，干净检出后的第一次配置前，需要先从仓库依赖树引导这两个工具： + +```bash +cd /path/to/cmvr-es + +CMVR_GRPC_PREFIX="$PWD/dependency/x86/third_party/grpc/v1.76.0" +CMVR_PROTOC="$CMVR_GRPC_PREFIX/bin/protoc" + +if [ ! -x "$CMVR_PROTOC" ]; then + CMVR_PROTOC="$CMVR_GRPC_PREFIX/bin/protoc-31.1.0" +fi + +test -x "$CMVR_PROTOC" +test -x "$CMVR_GRPC_PREFIX/bin/grpc_cpp_plugin" + +install -Dm755 "$CMVR_PROTOC" output/bin/protoc +install -Dm755 \ + "$CMVR_GRPC_PREFIX/bin/grpc_cpp_plugin" \ + output/bin/grpc_cpp_plugin + +export LD_LIBRARY_PATH="$CMVR_GRPC_PREFIX/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +``` + +该 `LD_LIBRARY_PATH` 仅用于首次配置和编译阶段找到仓库内的 Protobuf/gRPC 工具库。完成安装后，应用及安装库使用相对 RPATH，标准部署不需要全局设置它。 + +### 5.2 开发构建 + +```bash +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMVR_ARCH=x86 \ + -DBUILD_TESTING=ON \ + -DCMVR_MEDIA_SOURCE_HUB_BUILD_TESTS=ON + +cmake --build build -j"$(nproc)" +ctest --test-dir build --output-on-failure +cmake --install build +``` + +安装前缀由根 CMake 固定为仓库下的 `output/`。传入普通的 `-DCMAKE_INSTALL_PREFIX=...` 不会改变该目录。 + +安装结果的核心结构为： + +```text +output/ +├── bin/ +│ ├── cmvr_es +│ ├── config/ +│ └── model/ +├── lib/ +└── log/ # 首次产生日志时创建 +``` + +### 5.3 生产构建 + +生产构建可关闭无硬件测试目标： + +```bash +cmake -S . -B build-release \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMVR_ARCH=x86 \ + -DBUILD_TESTING=OFF \ + -DCMVR_MEDIA_SOURCE_HUB_BUILD_TESTS=OFF + +cmake --build build-release -j"$(nproc)" +cmake --install build-release +ldd -r output/bin/cmvr_es +``` + +> [!WARNING] +> 每次 `cmake --install` 都会删除并重新复制 `output/bin/config/` 和 `output/bin/model/`。不要把唯一一份生产配置、证书或现场模型直接维护在这两个目录中。 + +### 5.4 构建真实 QUIC 后端 + +MsQuic 是可选编译依赖。未找到 MsQuic 时，工程仍能构建和运行 gRPC，但启用 QUIC 任务会返回明确的“后端不可用”错误。 + +MsQuic 安装前缀需要至少包含： + +```text +/ +├── include/msquic.h +└── lib/libmsquic.so # 也可位于 lib64/ 或 bin/ +``` + +配置真实后端： + +```bash +cmake -S . -B build-quic \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMVR_ARCH=x86 \ + -DCMVR_ENABLE_MSQUIC_BACKEND=ON \ + -DCMVR_MSQUIC_ROOT=/absolute/path/to/msquic + +cmake --build build-quic -j"$(nproc)" +cmake --install build-quic +``` + +也可以将 MsQuic 放在 `dependency/x86/third_party/msquic//`，CMake 会自动搜索。配置阶段应看到 `MsQuic found`；若看到 `building QUIC edge in unavailable/stub mode`，则当前构建不能启动 QUIC 任务。 + +## 6. 配置说明 + +### 6.1 配置格式与加载规则 + +配置使用 Protobuf TextFormat，不是 JSON 或 YAML。配置 Schema 位于 [`protos/cmvr/config/`](protos/cmvr/config/)。 + +默认配置树： + +```text +cmvr_es.pb.txt +├── logger/logger.pb.txt +├── manager/device_manager.pb.txt +│ └── devices//*.pb.txt +└── manager/task_manager.pb.txt + └── tasks//*.pb.txt +``` + +关键入口： + +| 配置 | 作用 | +| --- | --- | +| [`cmvr_es.pb.txt`](cmvr-es/config/cmvr_es.pb.txt) | 根配置，引用日志、设备管理和任务管理配置 | +| [`device_manager.pb.txt`](cmvr-es/config/manager/device_manager.pb.txt) | 设备实例、类别、配置文件和启用状态 | +| [`task_manager.pb.txt`](cmvr-es/config/manager/task_manager.pb.txt) | gRPC、QUIC、触屏等任务及运行模式 | +| [`grpc_server_task.pb.txt`](cmvr-es/config/tasks/grpc_server_task/grpc_server_task.pb.txt) | gRPC 地址、端口和 reflection | +| [`quic_edge_task.pb.txt`](cmvr-es/config/tasks/quic_edge_task/quic_edge_task.pb.txt) | QUIC 网关、TLS、重连、心跳和媒体轨道 | +| [`logger.pb.txt`](cmvr-es/config/logger/logger.pb.txt) | 日志级别、路由、目录和轮转参数 | + +传入显式根配置时，业务配置、设备配置和 QUIC 证书的相对路径均以根配置文件所在目录解析： + +```bash +./output/bin/cmvr_es /etc/cmvr-es/cmvr_es.pb.txt +``` + +日志配置中的相对 `directory` 是相对于可执行文件目录解析，而不是配置根目录。默认值 `../log` 对应 `output/log/` 或部署后的 `/opt/cmvr-es/log/`。 + +### 6.2 默认无硬件开发配置 + +仓库默认配置适用于当前主机没有连接实际设备的场景： + +- 所有物理设备条目默认 `enable: false`； +- `touch_screen` 默认关闭； +- `quic_edge` 默认关闭； +- `grpc_server` 默认开启； +- gRPC 默认监听 `0.0.0.0:50052`，reflection 默认开启。 + +因此，首次启动应先验证进程和 gRPC 服务，再逐个开启设备。不要一次性启用全部设备。 + +### 6.3 启用设备 + +启用设备时需要同时满足： + +1. 对应设备集合配置中存在该实例； +2. `DeviceManager` 条目的 `id` 与设备配置中的 `id` 完全一致； +3. `DeviceManager` 条目的 `type` 与设备类别一致； +4. `enable: true`； +5. 厂商 SDK、权限、串口/CAN/USB 和网络参数已经准备完成。 + +推荐按以下顺序验证： + +1. 保持所有设备关闭，验证基础进程和 gRPC reflection； +2. 一次只开启一个设备； +3. 检查启动日志和对应设备的单次 RPC； +4. 再验证流式接口、并发访问和停止流程； +5. 最后启用 QUIC 媒体轨道。 + +### 6.4 启用 QUIC 注册与心跳 + +QUIC 必须同时在两处开启： + +1. 在 [`task_manager.pb.txt`](cmvr-es/config/manager/task_manager.pb.txt) 中设置 `quic_edge` 任务的 `enable: true`； +2. 在 [`quic_edge_task.pb.txt`](cmvr-es/config/tasks/quic_edge_task/quic_edge_task.pb.txt) 中设置 `quic_edge.enable: true`。 + +随后配置： + +- `server_host` 和 `server_port`； +- ALPN，v1 默认且要求精确匹配 `cmvr-quic-edge/1`； +- 唯一的 `node_id`； +- 心跳、ACK 超时和重连参数； +- 对外声明的 gRPC endpoint； +- CA、服务端名称以及可选的客户端证书和私钥。 + +媒体轨道可以为空。零轨道模式仍会完成节点注册、IP 上报和心跳，适合先联调在线状态。 + +正式环境应使用： + +```text +tls.allow_insecure: false +tls.ca_file: "certs/quic_gateway_ca.pem" +tls.server_name: "<与 server_host 匹配的服务端名称>" +``` + +客户端证书和私钥必须同时配置或同时省略。`allow_insecure: true` 只允许用于开发联调。 + +### 6.5 启用 QUIC 媒体轨道 + +启用媒体前必须先启用对应设备，然后在 QUIC 配置中增加唯一的 `track_id`： + +```protobuf +tracks { + track_id: 1 + source_kind: SOURCE_KIND_CAMERA + device_id: "right_hand_cam" + source_track_id: "right_hand_cam/video/color" + enable: true +} + +tracks { + track_id: 2 + source_kind: SOURCE_KIND_MICROPHONE + device_id: "mic1" + source_track_id: "mic1/audio/main" + enable: true +} +``` + +默认轨道命名规则： + +- 摄像头彩色流：`/video/color`； +- 麦克风主音频流：`/audio/main`。 + +`maximum_frame_bytes`、`maximum_datagram_bytes` 和 `datagram_send_queue_depth` 需要联合配置。一个完整帧的全部 DATAGRAM 分片必须能够作为批次进入发送队列，否则该帧会整体丢弃。 + +> [!NOTE] +> `grpc_endpoint_tls` 当前只描述“上报给平台的 gRPC 地址是否声称使用 TLS”，它不会自动让本项目 gRPC Server 启用 TLS。 + +## 7. 运行与测试 + +### 7.1 启动 + +使用安装目录内的默认配置： + +```bash +./output/bin/cmvr_es +``` + +使用外部配置根： + +```bash +./output/bin/cmvr_es /etc/cmvr-es/cmvr_es.pb.txt +``` + +程序不接受更多位置参数。正常停止使用 `Ctrl+C`、`SIGINT` 或 `SIGTERM`。 + +### 7.2 gRPC 快速验证 + +仓库提供 x86_64 版 grpcurl 资产： + +```bash +tar -xzf assets/grpcurl_1.8.7_linux_x86_64.tar.gz -C /tmp + +/tmp/grpcurl \ + -plaintext \ + 127.0.0.1:50052 \ + list + +/tmp/grpcurl \ + -plaintext \ + 127.0.0.1:50052 \ + list cmvr.api.ArmService +``` + +系统信息请求： + +```bash +/tmp/grpcurl \ + -plaintext \ + -d '{}' \ + 127.0.0.1:50052 \ + cmvr.api.SystemService/GetSystemInfo +``` + +reflection 正常只说明 gRPC Server 已启动。具体设备 RPC 仍可能因为设备未启用、初始化失败或硬件离线而失败。 + +### 7.3 无硬件自动测试 + +```bash +ctest --test-dir build --output-on-failure +``` + +当前登记到 CTest 的核心测试： + +| 测试 | 覆盖内容 | +| --- | --- | +| `hikvision_camera_callback_test` | Hikvision 回调生命周期和并发安全 | +| `media_source_hub_test` | 多订阅者、环形队列、丢帧和停止语义 | +| `quic_edge_protocol_test` | QUIC 控制帧、DATAGRAM 和 fake transport | +| `quic_edge_task_test` | QUIC 任务配置和生命周期 | + +仓库中其他以 `_test` 命名的程序可能需要真机、SDK、仿真环境或人工观察，不属于默认无设备验证。 + +### 7.4 进程烟雾测试 + +默认 gRPC 端口没有被占用时： + +```bash +timeout \ + --signal=TERM \ + --kill-after=2s \ + 5s \ + ./output/bin/cmvr_es +``` + +如果进程保持运行并在 5 秒后由 `timeout` 终止，退出码通常为 `124`，这是烟雾测试的预期结果。测试期间应同时检查日志中是否存在配置、设备初始化或端口绑定错误。 + +## 8. 生产部署 + +### 8.1 打包 + +部署时应分发整个 `output/`，不能只复制 `cmvr_es`： + +```bash +tar \ + --exclude='./log/*' \ + -C output \ + -czf cmvr-es-linux-x86_64.tar.gz \ + . +``` + +目标主机需要使用相同 CPU 架构，并具有兼容的 Linux/glibc 环境。归档会排除开发主机上的历史日志。安装产物使用相对 RPATH： + +- `bin/cmvr_es` 从自身目录和 `../lib` 查找共享库； +- `lib/` 中的共享库从自身目录查找依赖； +- 正常部署不依赖系统全局 `LD_LIBRARY_PATH`。 + +`output/` 仍会动态依赖目标系统的 libstdc++、PulseAudio/ALSA、X11、libudev、libusb，以及按设备启用的驱动或厂商服务。目标机应安装对应运行时依赖，并在启动服务前执行： + +```bash +ldd -r /opt/cmvr-es/bin/cmvr_es +``` + +命令输出中不应出现 `not found` 或未解析符号。 + +### 8.2 推荐目录 + +```text +/opt/cmvr-es/ +├── bin/ +├── lib/ +└── log/ + +/etc/cmvr-es/ +├── cmvr_es.pb.txt +├── logger/ +├── manager/ +├── devices/ +├── tasks/ +└── certs/ +``` + +示例： + +```bash +sudo install -d -m 0755 /opt/cmvr-es +sudo tar -C /opt/cmvr-es -xzf cmvr-es-linux-x86_64.tar.gz + +sudo install -d -m 0750 /etc/cmvr-es +sudo cp -a /opt/cmvr-es/bin/config/. /etc/cmvr-es/ +sudo install -d -m 0750 /etc/cmvr-es/certs + +sudo install -d -m 0750 /opt/cmvr-es/log +``` + +外部配置引用模型时，建议使用 `/opt/cmvr-es/bin/model/...` 的绝对路径，避免配置根切换后产生歧义。 + +### 8.3 systemd + +创建专用用户，并按实际硬件授予最小权限： + +```bash +sudo groupadd --system cmvr-es + +sudo useradd \ + --system \ + --gid cmvr-es \ + --home /opt/cmvr-es \ + --shell /usr/sbin/nologin \ + cmvr-es + +sudo usermod -aG video,audio,dialout cmvr-es +sudo chown -R cmvr-es:cmvr-es /opt/cmvr-es/log +sudo chown -R root:cmvr-es /etc/cmvr-es +sudo chmod -R g+rX /etc/cmvr-es + +# 按需安装 QUIC 信任链、客户端证书和私钥 +sudo install -o root -g cmvr-es -m 0640 \ + /secure/path/quic_gateway_ca.pem \ + /etc/cmvr-es/certs/quic_gateway_ca.pem + +sudo install -o root -g cmvr-es -m 0640 \ + /secure/path/cmvr_edge_cert.pem \ + /etc/cmvr-es/certs/cmvr_edge_cert.pem + +# 私钥只允许实际运行进程的用户读取 +sudo install -o cmvr-es -g cmvr-es -m 0600 \ + /secure/path/cmvr_edge_key.pem \ + /etc/cmvr-es/certs/cmvr_edge_key.pem +``` + +根据设备情况还可能需要配置 udev、CAN 网卡、USB 权限和厂商 SDK 服务。不要为了方便直接让进程长期以 root 身份运行。 + +`/etc/systemd/system/cmvr-es.service` 示例： + +```ini +[Unit] +Description=CMVR Edge System +Wants=network-online.target +After=network-online.target + +[Service] +Type=simple +User=cmvr-es +Group=cmvr-es +WorkingDirectory=/opt/cmvr-es/bin +ExecStart=/opt/cmvr-es/bin/cmvr_es /etc/cmvr-es/cmvr_es.pb.txt +Restart=on-failure +RestartSec=3 +KillSignal=SIGTERM +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target +``` + +启用服务： + +```bash +sudo systemctl daemon-reload +sudo systemctl enable --now cmvr-es +sudo systemctl status cmvr-es +journalctl -u cmvr-es -f +``` + +应用文件日志默认位于 `/opt/cmvr-es/log/cmvr_es.log`，终端路由的日志同时进入 journald。 + +### 8.4 网络与防火墙 + +| 方向 | 默认端口 | 协议 | 用途 | +| --- | ---: | --- | --- | +| Edge -> QUIC Gateway | `4433` | UDP / QUIC | 节点注册、心跳、IP 和实时媒体 | +| Platform -> Edge | `50052` | TCP / gRPC | 设备控制与状态查询 | +| Browser -> Gateway | 平台定义，通常 `443` | HTTPS / WebTransport / WebRTC | Web 控制和媒体播放 | + +如果平台根据心跳上报的 gRPC endpoint 回连边缘节点，平台网络必须能够访问该 TCP 地址。IP 上报成功不代表该地址一定具备路由、防火墙和 NAT 可达性。 + +## 9. 平台端接入 + +### 9.1 gRPC 接入 + +gRPC 契约位于 [`protos/cmvr/api/`](protos/cmvr/api/)，通用消息位于 [`protos/cmvr/msgs/`](protos/cmvr/msgs/) 和 [`protos/cmvr/common/`](protos/cmvr/common/)。 + +当前实际注册的服务： + +| 服务 | 主要用途 | +| --- | --- | +| `cmvr.api.SystemService` | 系统信息、状态和 StopAll | +| `cmvr.api.CameraService` | 相机状态、图片、录像和 RGB/Depth/RGBD 流 | +| `cmvr.api.SpeakerService` | 播放、流式输入、暂停、音量 | +| `cmvr.api.MicPhoneService` | 录音、音量和服务端音频流 | +| `cmvr.api.DexHandService` | 灵巧手控制、传感器数据和双向流 | +| `cmvr.api.BioHeadService` | 表情、状态、急停和双向流 | +| `cmvr.api.ArmService` | 上使能、运动、伺服、停止和运动学 | +| `cmvr.api.AgvService` | 导航、速度、地图、建图和地图流 | +| `cmvr.api.HlcService` | 触屏 / HLC 交互 | + +`test_service.proto` 存在于协议目录，但当前没有注册到 gRPC Server，平台不能把它当作在线服务。 + +以下 RPC 虽然存在于 proto，但当前实现不可用： + +- `cmvr.api.SystemService/UpdateParams` 固定返回业务失败，提示该能力不再支持； +- `cmvr.api.ArmService/getPoseMatrix` 返回 gRPC `UNIMPLEMENTED`； +- `cmvr.api.ArmService/computeForwardKinematics` 返回 gRPC `UNIMPLEMENTED`。 + +平台生成客户端后仍应以服务端实现状态为准，不能把“proto 中存在”直接等同于“运行时已支持”。 + +流式方向： + +- Camera RGB、Depth、RGBD：双向流； +- DexHand Sensor、BioHead Expression：双向流； +- Microphone Audio、AGV Map：服务端流； +- Speaker Audio：客户端流。 + +Java 平台建议通过 Gradle/Maven 的 Protobuf 插件生成消息类和 grpc-java stub，并在平台仓库中锁定 `protoc`、`protoc-gen-grpc-java`、`protobuf-java` 和 grpc-java 版本。生成时应保持 `protos/` 为 import 根目录。 + +当前 proto 没有统一声明 `java_package` 和 `java_multiple_files`。平台接入时不要自行修改包名后仍声称与线协议一致；如要补充 Java options，应由边缘端与平台端在同一协议变更中完成并重新生成两端代码。 + +平台调用流程建议： + +1. 通过 QUIC 注册/心跳维护节点在线表； +2. 保存最新 `node_id`、`session_id`、`observed_source_ip`、本地接口和 advertised gRPC endpoint； +3. 按平台网络策略选择实际可达的 gRPC 地址； +4. 建立并复用 gRPC channel； +5. 为控制请求设置 deadline、幂等策略、错误映射和审计； +6. 不要因为 QUIC 在线就假设设备本身在线。 + +> [!WARNING] +> 当前 gRPC Server 使用 `grpc::InsecureServerCredentials()`，没有 TLS、认证和授权。只能部署在可信内网、VPN、服务网格或受保护的网关后方。 + +### 9.2 Java QUIC Gateway 接入 + +边缘端是 QUIC client，平台网关是 QUIC server。本仓库没有提供 Java 网关实现。 + +Java 网关只需要为 QUIC 控制面生成普通 Protobuf Java 类，不需要生成 gRPC stub： + +```bash +mkdir -p platform-gateway/src/main/java + +output/bin/protoc \ + -I protos \ + --java_out=platform-gateway/src/main/java \ + protos/cmvr/quic_edge/v1/quic_edge.proto +``` + +完整协议见 [`QUIC edge v1 wire format`](protos/cmvr/quic_edge/v1/README.md)。网关至少需要实现以下行为。 + +#### 可靠控制流 + +边缘端建立 QUIC 连接后打开一个双向可靠流。消息格式为： + +```text +uint32_be protobuf_length +protobuf EdgeControlEnvelope +``` + +接收端不能把一次 QUIC stream callback 当作一条完整消息，必须缓存并按长度前缀拆包。长度不包含 4 字节前缀。 + +会话顺序： + +1. Edge 首先发送 `NodeRegisterRequest`； +2. Gateway 返回 `accepted=true` 且 `session_id` 非空的 `NodeRegisterResponse`； +3. Edge 开始发送 `NodeHeartbeat`； +4. Gateway 返回相同 `session_id` 和精确 `acknowledged_sequence` 的 `NodeHeartbeatAck`； +5. 有媒体轨道时，Edge 先调用可靠流发送 `MediaSessionOpen` 和 `MediaTrackDescriptor`，再调用 DATAGRAM 发送媒体； +6. 编码参数或 generation 变化时，Edge 同样先发送新的 descriptor，再发送对应 generation 的 DATAGRAM。 + +QUIC reliable stream 与 DATAGRAM 之间没有跨通道到达顺序保证。即使 Edge 按上述顺序调用发送，媒体 DATAGRAM 仍可能先于 session/descriptor 元数据到达 Gateway。Gateway 对未知 `session_epoch` 或 `codec_generation_token` 的 DATAGRAM 必须限量暂存或直接丢弃，不能无界等待，也不能假设可靠元数据一定先到。 + +实现细节： + +- `protocol_version` 当前为 `1`； +- `message_sequence` 在每个方向独立严格递增，允许有间隔； +- Edge 当前第一条控制消息的 sequence 是 `0`，网关不能假设从 `1` 开始； +- 默认控制响应超时为 1 秒，注册或心跳 ACK 缺失/不匹配会触发重连； +- 每次心跳包含最新接口地址和 advertised gRPC endpoint； +- `observed_source_ip` 必须从已认证的 QUIC peer 地址推导，不能复制客户端上报字段； +- 当前 Edge 接收方向实现支持 `NodeRegisterResponse`、`NodeHeartbeatAck` 和 `ProtocolError`； +- 不要依赖 Gateway 下发 `MediaSessionClose` 来控制 Edge，本版本尚未实现该入站行为。 + +#### 媒体 DATAGRAM + +每个媒体 DATAGRAM 以固定 64 字节、网络字节序的 `CMQD` header 开始。字段和偏移详见 [`QUIC edge v1 wire format`](protos/cmvr/quic_edge/v1/README.md#media-datagram-wire-format)。 + +网关需要： + +- 以 `(session_epoch, track_id, frame_sequence)` 作为重组键； +- 校验 magic、版本、header size、fragment count、offset 和 frame size； +- 在播放 deadline 到达时丢弃不完整帧，不等待旧帧阻塞新帧； +- 新的 `session_epoch` 到来时清除旧连接的残留分片； +- 根据 `MediaTrackDescriptor` 和 `codec_generation_token` 选择解码参数； +- 容忍 DATAGRAM 先于可靠元数据到达，对未知 session/generation 有界暂存或丢弃； +- 根据 keyframe 和 discontinuity 标志重置解码/转封装状态； +- 为每个节点和轨道设置内存、帧大小、并发和超时上限。 + +当前媒体模型支持 H.264、H.265、Opus、AAC 和 PCM S16LE。实际可用编码取决于具体设备和配置。 + +### 9.3 浏览器接入 + +浏览器不能直接连接 `cmvr-quic-edge/1` 自定义 ALPN，也不承担节点注册服务。推荐拓扑： + +```text +cmvr_es + │ + ├── QUIC reliable stream + DATAGRAM ──> Platform QUIC Gateway + │ │ + │ ├── WebTransport + │ ├── WebRTC + │ ├── WebSocket + MSE + │ └── HTTP/HLS 等平台接口 + │ + └── gRPC control <──────────────────── Platform Service + │ + └── REST / WebSocket / gRPC-Web + │ + Browser +``` + +平台网关负责： + +- 终止 CMVR QUIC 和维护节点在线状态； +- 重组音视频帧； +- 根据浏览器兼容性转封装或转码； +- 对 Web 用户执行认证、授权、租户隔离和设备 ACL； +- 处理慢浏览器、断线、关键帧请求和背压； +- 将浏览器控制请求转换为平台到边缘端的 gRPC； +- 向浏览器隐藏设备密码、证书和内部 gRPC 地址。 + +H.265、AAC 等格式的浏览器支持度并不统一。平台不能假设收到的原始编码能够在所有浏览器中直接播放，应根据目标浏览器选择 WebRTC codec、fMP4/MSE、转码或降级策略。 + +### 9.4 平台联调清单 + +建议按以下顺序联调： + +1. Edge 保持零媒体轨道，验证 UDP、TLS、ALPN 和注册； +2. 验证 `observed_source_ip`、本地接口列表和 gRPC endpoint 更新； +3. 验证精确 heartbeat ACK、超时和重连； +4. 使用 grpcurl 或 Java client 验证平台到 Edge 的 gRPC 可达性； +5. 启用一个 fake/真实摄像头轨道，验证 descriptor 和 DATAGRAM 重组； +6. 模拟丢包、乱序、重复、超时和 Edge 重连； +7. 再启用音频、多轨道和多节点； +8. 最后接入浏览器转封装、权限和运维监控。 + +平台应至少监控： + +- 节点在线数、session 变化和最后心跳时间； +- QUIC 握手/注册失败原因； +- 心跳 RTT、ACK 超时和重连次数； +- 每轨道 DATAGRAM、完整帧、丢帧和重组超时； +- descriptor/generation 变化； +- gRPC 调用时延、错误码和设备离线状态； +- 网关内存、队列深度和浏览器订阅数。 + +## 10. 开发扩展指南 + +### 10.1 新增同类设备后端 + +以新增摄像头后端为例： + +1. 在对应 config proto 的 `oneof backend` 中增加后端配置； +2. 实现已有摄像头抽象接口，不在设备类中加入 gRPC/QUIC 连接状态； +3. 在类别工厂中增加具体后端创建分支； +4. 增加 CMake target 和 SDK 查找； +5. 在设备集合配置中增加实例； +6. 在 `DeviceManager` 配置中增加相同 ID 的条目； +7. 实时媒体设备补齐编码描述、时间戳、序列号、generation 和关键帧能力； +8. 增加不依赖真机的生命周期与并发测试，再进行真机测试。 + +设备抽象应保持薄且与具体控制器协议无关。控制器专有结构、错误码和报文字段应留在具体后端。 + +### 10.2 新增设备类别 + +除上述工作外，还需要： + +- 增加新的抽象接口与类别工厂； +- 扩展 `DeviceConfigEntry::DeviceType`； +- 在全局 `DeviceFactory` 注册 creator； +- 在 `DeviceManager` 中增加类型实例化和日志名称； +- 如需平台调用，增加 API proto、gRPC service 实现，并在 `GrpcServerTask` 中注册； +- 明确设备关闭、急停和进程退出时的生命周期语义。 + +### 10.3 新增任务 + +1. 增加任务配置 proto 和默认配置； +2. 实现 `Task` 接口； +3. 明确任务是 `PERIODIC_STEP` 还是 `BLOCKING_SERVICE`； +4. 扩展 `TaskType` 并注册 TaskFactory creator； +5. 在 `TaskManager` 默认配置中增加任务； +6. 增加 CMake target 和 init/start/stop 测试。 + +### 10.4 新增媒体传输协议 + +后续接入 WebRTC、SRT 或其他协议时： + +1. 直接订阅 `MediaSourceHub`； +2. 每个消费线程创建独立 `Subscription`； +3. 将 `TrackDescriptor` 映射为目标协议的 codec metadata； +4. 根据 dropped count、generation 和 frame 标志传播 discontinuity； +5. H.264/H.265 丢帧后请求关键帧； +6. 协议断开或停止时及时 reset subscription； +7. 不要修改摄像头或麦克风接口来保存协议连接状态； +8. 增加慢消费者、环形队列覆盖、设备重启和并发停止测试。 + +这一约束保证 MediaSourceHub 能快速适配其他协议，同时设备层保持稳定。 + +### 10.5 修改 Proto 的兼容性要求 + +- 不复用已经发布字段的 tag； +- 删除字段时使用 `reserved`； +- 优先增加可选字段，不随意改变现有语义； +- gRPC API 和 QUIC v1 线协议分别维护版本兼容； +- 修改 QUIC DATAGRAM 固定头时必须提升协议版本并同步平台网关； +- 提交前重新生成并验证 C++、Java 代码； +- 协议文档、示例配置和自动测试必须与代码同一提交更新。 + +## 11. 安全说明 + +生产环境必须关注： + +- gRPC 当前是明文且无认证授权，不能直接暴露到公网； +- QUIC 正式环境必须验证服务端证书，必要时使用 mTLS； +- `allow_insecure` 仅用于开发； +- QUIC schema 当前没有业务 token 或设备 ACL，应用级身份和授权需要平台设计； +- 配置可以包含设备账号、密码、私钥和生产地址，不要提交真实凭据； +- 私钥和生产配置建议权限为 `0600`，由专用运行用户读取； +- 平台应限制允许控制的用户、节点、设备和 RPC； +- 机械臂、AGV 等控制必须设置 deadline、速度/位置限制、急停和审计； +- 设备控制安全不能只依赖网络协议成功，仍需设备侧限位和安全控制器。 + +## 12. 已知限制与排障 + +### 12.1 首次配置提示找不到 protoc 或 grpc_cpp_plugin + +确认已执行[首次构建工具引导](#51-首次构建工具引导)，并在同一 shell 中保留 gRPC 依赖库的 `LD_LIBRARY_PATH` 后重新配置和编译。 + +### 12.2 CMake 显示 MsQuic not found + +这不会影响 gRPC 和 fake transport 测试，但当前产物不能启动 QUIC 任务。检查： + +- `CMVR_ENABLE_MSQUIC_BACKEND=ON`； +- `CMVR_MSQUIC_ROOT` 使用绝对路径； +- `/include/msquic.h` 存在； +- `/lib/libmsquic.so`、`lib64/` 或 `bin/` 中存在库； +- 重新配置时清理了错误的 CMake cache。 + +### 12.3 启动后 gRPC connection refused + +检查： + +```bash +ss -lntp | grep 50052 +journalctl -u cmvr-es -n 200 --no-pager +``` + +常见原因： + +- `grpc_server` 任务被关闭； +- 端口被其他进程占用； +- 主进程因配置或设备初始化错误提前退出； +- 平台访问的是不可路由的心跳上报地址； +- 主机或容器防火墙未开放 TCP 50052。 + +### 12.4 reflection 正常但设备 RPC 失败 + +reflection 只验证服务注册。继续检查： + +- 设备配置和 `DeviceManager` 条目是否都启用； +- 两处设备 ID 是否一致； +- 设备 SDK、串口、CAN、USB 或网络是否可用； +- 运行用户是否具备权限； +- 日志中设备 `init()` 是否成功。 + +### 12.5 QUIC 持续重连 + +按顺序检查： + +1. Edge 到 Gateway 的 UDP 端口是否可达； +2. ALPN 是否精确为 `cmvr-quic-edge/1`； +3. CA、服务端证书和 `server_name` 是否匹配； +4. Gateway 是否返回 accepted 且非空 session ID； +5. heartbeat ACK 的 session 和 sequence 是否精确匹配； +6. Gateway 是否在默认 1 秒控制响应超时内回复； +7. 是否错误地把 stream callback 边界当作消息边界。 + +### 12.6 找不到共享库或出现未解析符号 + +检查： + +```bash +ldd output/bin/cmvr_es +ldd -r output/bin/cmvr_es +find output/lib -maxdepth 1 -type f -name '*.so*' -print +``` + +部署时确认复制了完整 `output/`，并保持 `bin/` 与 `lib/` 的相对位置。不要混用不同构建、不同架构或不兼容 glibc 的产物。 + +### 12.7 安装后现场配置被覆盖 + +`cmake --install` 会重建 `output/bin/config/` 和 `output/bin/model/`。生产配置应保存在 `/etc/cmvr-es/` 等外部目录，并通过命令行显式传入根配置。 + +### 12.8 Git submodule 初始化失败 + +当前仓库不应执行旧文档中的 `git submodule update --init --recursive`。构建依赖来自已跟踪的 `dependency//third_party/`；`.gitmodules` 与 `assets/toppra` 的历史不一致需要单独整理。 + +### 12.9 当前未提供的能力 + +本仓库当前不包含： + +- Java QUIC Gateway； +- 浏览器播放器或 Web 控制前端； +- Dockerfile / 容器镜像； +- 可直接安装的 systemd unit； +- 标准 gRPC health service； +- gRPC TLS、认证和授权； +- 完整可复现的 ARM 构建。 + +这些能力不能仅通过修改配置开启，需要在对应工程中实现和验证。 + +## License + +许可证内容见 [`LICENSE`](LICENSE)。同时请遵守 `dependency/` 中各三方组件和厂商 SDK 的独立许可要求。 diff --git a/cmake/FindMsQuic.cmake b/cmake/FindMsQuic.cmake new file mode 100644 index 00000000..4af6ec6f --- /dev/null +++ b/cmake/FindMsQuic.cmake @@ -0,0 +1,50 @@ +# Locate an optional MsQuic installation without pulling in a second TLS runtime. +# +# The project supports either: +# -DCMVR_MSQUIC_ROOT=/path/to/msquic/prefix +# or a package placed below: +# dependency//third_party/msquic// +# +# Result: +# MsQuic_FOUND +# MsQuic::msquic + +include_guard(GLOBAL) + +set(CMVR_MSQUIC_ROOT "" CACHE PATH "MsQuic installation prefix") + +set(_cmvr_msquic_hints) +if(CMVR_MSQUIC_ROOT) + list(APPEND _cmvr_msquic_hints "${CMVR_MSQUIC_ROOT}") +endif() + +if(DEFINED PROJECT_SOURCE_DIR AND DEFINED ARCH) + file(GLOB _cmvr_msquic_bundled_roots LIST_DIRECTORIES true + "${PROJECT_SOURCE_DIR}/dependency/${ARCH}/third_party/msquic/*") + list(APPEND _cmvr_msquic_hints + "${PROJECT_SOURCE_DIR}/dependency/${ARCH}/third_party/msquic" + ${_cmvr_msquic_bundled_roots}) +endif() + +find_path(MsQuic_INCLUDE_DIR + NAMES msquic.h + HINTS ${_cmvr_msquic_hints} + PATH_SUFFIXES include) + +find_library(MsQuic_LIBRARY + NAMES msquic libmsquic + HINTS ${_cmvr_msquic_hints} + PATH_SUFFIXES lib lib64 bin) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(MsQuic + REQUIRED_VARS MsQuic_INCLUDE_DIR MsQuic_LIBRARY) + +if(MsQuic_FOUND AND NOT TARGET MsQuic::msquic) + add_library(MsQuic::msquic UNKNOWN IMPORTED) + set_target_properties(MsQuic::msquic PROPERTIES + IMPORTED_LOCATION "${MsQuic_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${MsQuic_INCLUDE_DIR}") +endif() + +mark_as_advanced(MsQuic_INCLUDE_DIR MsQuic_LIBRARY) diff --git a/cmvr-es/CMakeLists.txt b/cmvr-es/CMakeLists.txt index eb16b40f..64a5894e 100644 --- a/cmvr-es/CMakeLists.txt +++ b/cmvr-es/CMakeLists.txt @@ -6,7 +6,10 @@ add_subdirectory(hardware) add_subdirectory(algorithms) add_subdirectory(devices) add_subdirectory(manager/device_manager) +add_subdirectory(manager/media_source_hub) +add_subdirectory(service/quic_edge) add_subdirectory(task) +add_subdirectory(task/quic_edge_task) add_subdirectory(manager/task_manager) add_subdirectory(service) add_subdirectory(simulate) diff --git a/cmvr-es/algorithms/README.md b/cmvr-es/algorithms/README.md new file mode 100644 index 00000000..90e64f19 --- /dev/null +++ b/cmvr-es/algorithms/README.md @@ -0,0 +1,125 @@ +# Algorithms 模块开发指南 + +`algorithms/` 保存与具体厂商协议无关的运动学、规划、控制和感知算法。算法接受通用类型或显式接口输入，不应直接解析设备报文，也不应承担 gRPC/QUIC 传输职责。 + +返回[项目总览](../../README.md)。 + +## 当前结构 + +| 目录 | 主要能力 | 主要 CMake target | +| --- | --- | --- | +| `kinematics/ik_solver/` | Pinocchio DLS/QP、SRS、LAWBA 逆运动学 | `cmvr_es::ik_solver` | +| `motion_planner/base_motion/` | TOPPRA、S 曲线、笛卡尔速度限制 | `cmvr_es::base_motion` | +| `motion_planner/arm_motion/` | MoveJ、MoveL、SpeedL 机械臂规划 | `cmvr_es::algorithms::arm_motion` | +| `controllers/` | PID、IBVS、笛卡尔速度控制 | `cmvr_es::algorithms::controller`、`cmvr_es::algorithms::arm_control` | +| `perception/` | AprilTag 和视觉定位 | `cmvr_es::perception` | + +顶层入口是 [`CMakeLists.txt`](CMakeLists.txt)。 + +## 依赖边界 + +- 算法层可以依赖 `common/`、Eigen、Pinocchio、OSQP、TOPPRA、OpenCV、ViSP 等； +- 不包含串口、CAN、HTTP 或厂商 SDK 协议处理； +- 不启动 gRPC/QUIC 服务或管理设备生命周期； +- 不从算法内部读取全局配置文件，构造或 `configure` 时显式传入配置； +- 可复用算法不应主动取得 `DeviceManager` 单例。 + +当前部分 controller target 仍链接 `device_manager`，这是现有耦合。新增算法应优先通过参数、回调或窄接口注入设备状态，避免继续扩大该依赖。 + +## 扩展已有算法类别 + +### 1. 定义或复用抽象接口 + +常用接口： + +- [`IKSolver`](kinematics/ik_solver/common/include/ik_solver.h) +- [`JointMotionPlanner`](motion_planner/arm_motion/joint_motion/joint_motion_planner.h) +- [`CartesianMotionPlanner`](motion_planner/arm_motion/cartesian_motion/cartesian_motion_planner.h) + +接口应明确： + +- 输入输出单位和坐标系； +- 是否修改内部状态； +- 是否线程安全； +- 失败时输出是否保持不变； +- 是否支持实时循环，以及最大允许耗时。 + +### 2. 增加配置 + +在 [`../../protos/README.md`](../../protos/README.md) 指导下： + +1. 为算法增加独立配置 message； +2. 在所属 `oneof algorithm` 中增加新字段和新 tag； +3. 不复用已发布 tag； +4. 为迭代次数、容差、速度和加速度设置有效范围； +5. 在默认设备配置中给出显式参数。 + +### 3. 实现与工厂注册 + +将实现放在对应类别子目录，并修改实际工厂： + +- IK：[`ik_solver_factory.h`](kinematics/ik_solver/ik_solver_factory.h) +- MoveJ：[`joint_motion_planner_factory.h`](motion_planner/arm_motion/joint_motion/joint_motion_planner_factory.h) +- MoveL / SpeedL：[`cartesian_motion_planner_factory.h`](motion_planner/arm_motion/cartesian_motion/cartesian_motion_planner_factory.h) + +工厂失败应返回 `nullptr` 并记录清晰原因，不能静默回退到另一个算法。MoveL 和 SpeedL 的实现必须保持配置组合一致。 + +### 4. 更新 CMake + +- 将实现 `.cpp` 加入对应 library； +- 使用项目已有 alias target； +- 通过 `target_include_directories` 暴露公共头； +- 将依赖放入使用它的最小 target； +- 测试源文件不能加入生产共享库； +- 新增三方依赖时同步根依赖发现逻辑和 `request.txt`。 + +## 数值与机器人语义 + +算法扩展至少需要明确： + +- 关节位置单位为 rad，速度为 rad/s； +- 笛卡尔平移为 m，旋转和角速度为 rad； +- base、tool、world、user frame 的转换方向； +- URDF base frame、tip frame 和关节顺序； +- 位置、速度、加速度和 jerk 限制； +- 奇异点、不可达目标和求解超时行为； +- measured state 与算法内部 seed 的更新时机。 + +IK 在求解前应使用真实关节角更新 seed。MoveL 连续求解时，应使用上一步解更新下一步状态，不能一直使用初始状态。 + +## 测试要求 + +每个新算法至少覆盖： + +1. 正常输入； +2. 空输入、自由度不匹配和 NaN/Inf； +3. 关节限位与速度限制； +4. 不可达目标和不收敛； +5. 坐标系转换； +6. 确定性和重复调用； +7. 若用于实时控制，统计最坏执行时间； +8. 与一个已知模型或离线参考结果对比。 + +当前不少算法测试只通过 `add_executable()` 构建，没有登记到 CTest。新增无设备测试应放在 `BUILD_TESTING` 条件内，并使用 `add_test()`；需要图形界面、RealSense 或 MuJoCo 的测试应明确标为集成测试，不得阻塞默认无设备测试。 + +## 新增算法类别 + +如果现有类别无法承载： + +1. 在 `algorithms//` 新建目录； +2. 定义协议无关抽象接口； +3. 定义配置 Proto 和工厂； +4. 提供单独 CMake library 与 `cmvr_es::...` alias； +5. 在 [`algorithms/CMakeLists.txt`](CMakeLists.txt) 添加子目录； +6. 由设备或任务层注入使用，不让算法反向控制服务层； +7. 添加无设备单元测试和真实设备/仿真集成测试。 + +## 提交检查 + +- [ ] 厂商协议没有进入算法接口 +- [ ] 单位、坐标系和关节顺序明确 +- [ ] 工厂已注册且配置组合经过校验 +- [ ] 不可达、超时和数值异常可观测 +- [ ] 测试没有被编入生产共享库 +- [ ] 无设备测试已登记到 CTest +- [ ] 实时路径没有日志洪泛和无界内存分配 diff --git a/cmvr-es/common/README.md b/cmvr-es/common/README.md new file mode 100644 index 00000000..cf60bb8d --- /dev/null +++ b/cmvr-es/common/README.md @@ -0,0 +1,111 @@ +# Common 模块开发指南 + +`common/` 保存可被设备、算法、管理器和协议层复用的基础能力。这里适合放稳定、协议无关、厂商无关的类型与工具，不适合放设备连接、业务服务或任务调度逻辑。 + +返回[项目总览](../../README.md)。 + +## 目录职责 + +| 目录 | 职责 | +| --- | --- | +| `base/` | 日志、基础常量、gRPC 辅助函数和线程安全缓冲区 | +| `config/` | 配置根目录解析和 Proto Text 配置加载 | +| `io/` | Protobuf 二进制与 TextFormat 文件读写 | +| `math/` | 坐标变换、关节限制、QP 和运动数学 | +| `media/` | 协议无关媒体模型以及 FFmpeg 采集、编码、写文件能力 | +| `types/` | 跨后端共享的领域类型，例如 AGV、机械臂和几何类型 | +| `vision/` | 图像显示、投影等视觉辅助代码 | + +## 依赖边界 + +新增公共组件时应遵守： + +- 不依赖 `service/`、`task/` 或具体厂商设备实现； +- 不保存 gRPC/QUIC 连接、session 或客户端状态； +- 通用类型不包含厂商报文字段、端口号和私有错误码； +- 需要调用设备的逻辑应放在 manager adapter、service 或 task； +- 需要第三方库的 `.cpp` 组件应通过明确的 CMake target 暴露依赖； +- 避免在公共头文件中使用全局 `using namespace` 或引入大体量实现头。 + +当前 `common` 共享库目标是 `cmvr_es::common`，日志是独立目标 `cmvr_es::logging`。新增 `.cpp` 文件时，需要更新 [`CMakeLists.txt`](CMakeLists.txt) 或对应子目录 CMake；纯头文件不需要加入 `add_library` 源文件列表。 + +## 新增共享类型 + +1. 选择 `types//` 或已有领域文件； +2. 类型使用明确单位，例如米、弧度、秒、纳秒； +3. 为容器长度、自由度和数值范围提供校验函数； +4. 保持控制器无关，将厂商字段转换为通用枚举或结果； +5. 确认不会迫使所有调用方引入设备 SDK； +6. 增加边界值和错误输入测试。 + +AGV 通用类型应参考 [`types/agv/agv_types.h`](types/agv/agv_types.h)，机械臂通用类型应参考 [`types/arm/arm_types.h`](types/arm/arm_types.h)。不要为了一个具体控制器把协议结构塞回 `abstract_*.h`。 + +## 媒体模型 + +[`media/media_frame.h`](media/media_frame.h) 中的 `TrackDescriptor`、`MediaFrame` 及 payload 在构造后不可变，可被多个协议消费者共享。 + +扩展媒体字段时需要保持： + +- `TrackDescriptor::generation` 非零，编码参数变化时创建新 descriptor； +- PTS、DTS 和 duration 使用 descriptor 的 `time_base`； +- `capture_time_ns` 使用单调时钟，供节奏控制和延迟统计； +- `capture_utc_ns` 只作为可选墙上时间，不能用于计算持续时间； +- H.264/H.265 明确 `ANNEX_B` 或 `AVCC`； +- AAC、Opus、PCM 明确 payload format、采样率和声道数； +- 不把 QUIC、gRPC 或浏览器专有字段加入通用帧。 + +设备媒体接入流程见 [`../manager/README.md`](../manager/README.md) 的 MediaSourceHub 章节。 + +## 环形队列选择 + +[`base/ring_buffer.h`](base/ring_buffer.h) 当前包含三类缓冲区： + +| 类型 | 使用场景 | 重要约束 | +| --- | --- | --- | +| `RingBuffer` | 只需要保存最近 N 项并批量读取 | 覆盖最旧项，没有阻塞读取 | +| `SPMCRingBuffer` | 历史单生产者场景 | 独立 `reader_tail` 只能由一个线程拥有 | +| `BroadcastFrameRing` | 新的媒体或广播式多消费者场景 | 每个消费者使用独立 Cursor，保存不可变共享对象 | + +新的实时多消费者模块优先使用 `BroadcastFrameRing`： + +- capacity 必须大于零； +- 同一 Cursor 不得被多个线程同时读取或移动； +- 慢消费者落后时会跳到最旧保留项，并得到精确 dropped count； +- `reset()` 开启新 generation，旧 Cursor 在下一次成功读取时看到变化； +- `close()` 唤醒等待者，关闭后不能继续发布； +- 不要先读取 head 再无锁读取槽位，应使用队列提供的原子读取接口。 + +## 配置和文件路径 + +[`config/config_files.h`](config/config_files.h) 提供： + +- `resolveConfigFile()`：相对根配置目录解析业务配置； +- `resolveResourceFile()`：在配置根及其父目录中查找模型等资源； +- `loadConfigFile()` / `saveConfigFile()`：读写 Proto Text 配置。 + +进程启动后配置根由 `main.cpp` 设置。公共组件不应自行使用当前工作目录拼接配置路径。 + +[`io/proto_file_io.h`](io/proto_file_io.h) 写出的 TextFormat 文件权限为 `0600`。保存运行时配置前，应确认目标目录存在，并避免把生产密钥写入仓库。 + +## 新增公共组件 + +1. 确认能力确实会被两个及以上模块复用； +2. 定义最小 API 和所有权、线程安全、错误语义； +3. 将头文件放入合适子目录，将实现放入相邻 `.cpp`； +4. 更新 CMake target 和 `target_link_libraries`； +5. 不使用未声明的传递依赖； +6. 增加无设备单元测试； +7. 对并发组件增加关闭、超时、覆盖、取消和析构测试； +8. 使用 ASan/TSan 时检查生命周期和数据竞争。 + +推荐测试目标放在组件相邻的 `tests/`，并在 `BUILD_TESTING` 下通过 `add_test()` 登记。仅创建 `_test` 可执行文件不会自动进入 CTest。 + +## 提交检查 + +- [ ] API 不依赖具体设备或传输协议 +- [ ] 公共类型有明确单位和有效性规则 +- [ ] 所有权及线程安全写入注释 +- [ ] 新增 `.cpp` 和依赖已经加入 CMake +- [ ] 缓冲区关闭能够唤醒等待线程 +- [ ] 不记录密码、私钥或大块媒体 payload +- [ ] 无设备测试可以在开发主机运行 diff --git a/cmvr-es/common/base/ring_buffer.h b/cmvr-es/common/base/ring_buffer.h index bf31679c..32e0479c 100644 --- a/cmvr-es/common/base/ring_buffer.h +++ b/cmvr-es/common/base/ring_buffer.h @@ -6,16 +6,25 @@ #define CMVR_ES_RING_BUFFER_H #pragma once +#include +#include +#include #include +#include #include -#include -#include #include +#include +#include +#include template class RingBuffer { public: - explicit RingBuffer(size_t capacity) : capacity_(capacity) {} + explicit RingBuffer(size_t capacity) : capacity_(capacity) { + if (capacity_ == 0) { + throw std::invalid_argument("RingBuffer capacity must be greater than zero"); + } + } void push(const T& item) { std::lock_guard lock(mutex_); @@ -50,47 +59,49 @@ template class SPMCRingBuffer { public: explicit SPMCRingBuffer(size_t capacity) - : buffer_(capacity), capacity_(capacity), - head_(0), tail_(0) {} + : buffer_(capacity), capacity_(capacity) { + if (capacity_ == 0) { + throw std::invalid_argument("SPMCRingBuffer capacity must be greater than zero"); + } + } // 写入操作（仅支持单个生产者） void push(const T& item) { - size_t head = head_.load(std::memory_order_relaxed); - size_t tail = tail_.load(std::memory_order_acquire); - buffer_[head % capacity_] = item; - head = head + 1; - head_.store(head, std::memory_order_release); - if (head - tail >= capacity_) { - // 队列满，覆盖最旧的数据 - tail_.store(tail + 1, std::memory_order_release); + { + std::lock_guard lock(mutex_); + buffer_[head_ % capacity_] = item; + ++head_; + if (head_ - tail_ > capacity_) { + // 队列满，覆盖最旧的数据 + tail_ = head_ - capacity_; + } } + condition_.notify_all(); } // 单消费者使用（内部 tail_） std::optional pop() { - size_t tail = tail_.load(std::memory_order_relaxed); - size_t head = head_.load(std::memory_order_acquire); - if (tail >= head) return std::nullopt; - T value = buffer_[tail % capacity_]; - tail_.store(tail + 1, std::memory_order_release); + std::lock_guard lock(mutex_); + if (tail_ >= head_) return std::nullopt; + T value = buffer_[tail_ % capacity_]; + ++tail_; return value; } - std::optional getLast() { - size_t tail = tail_.load(std::memory_order_relaxed); - size_t head = head_.load(std::memory_order_acquire); - if (tail >= head) return std::nullopt; - T value = buffer_[head_ % capacity_]; - return value; + std::optional getLast() const { + std::lock_guard lock(mutex_); + if (tail_ >= head_) return std::nullopt; + return buffer_[(head_ - 1) % capacity_]; } - // 多消费者使用（每个读者独立维护 reader_tail） + // 多消费者使用（每个读者独立维护 reader_tail）。同一个 reader_tail 只能由 + // 一个消费线程拥有，且不要把该游标与无参 pop() 的共享 tail_ 混合作为同一路读取。 std::optional pop(size_t& reader_tail) const { - size_t head = head_.load(std::memory_order_acquire); - if (reader_tail >= head) return std::nullopt; - if (head > reader_tail + capacity_) { + std::lock_guard lock(mutex_); + if (reader_tail >= head_) return std::nullopt; + if (reader_tail < tail_) { // 数据已被覆盖，跳过无效读取区间 - reader_tail = head - capacity_; + reader_tail = tail_; return std::nullopt; } T value = buffer_[reader_tail % capacity_]; @@ -98,16 +109,52 @@ public: return value; } + // 在同一次加锁中把独立读游标跳到当前最新元素并读取，避免先 getHead() + // 再 pop() 时被高速覆盖造成的检查/读取竞态。 + std::optional getLatest(size_t& reader_tail) const { + std::lock_guard lock(mutex_); + if (tail_ >= head_) { + return std::nullopt; + } + reader_tail = head_ - 1; + T value = buffer_[reader_tail % capacity_]; + ++reader_tail; + return value; + } + + template + std::optional waitPop( + size_t& reader_tail, + const std::chrono::duration& timeout) const { + std::unique_lock lock(mutex_); + condition_.wait_for(lock, timeout, [&] { return reader_tail < head_; }); + if (reader_tail >= head_) { + return std::nullopt; + } + if (reader_tail < tail_) { + reader_tail = tail_; + } + if (reader_tail >= head_) { + return std::nullopt; + } + T value = buffer_[reader_tail % capacity_]; + ++reader_tail; + return value; + } + size_t size() const { - return head_.load(std::memory_order_acquire) - tail_.load(std::memory_order_acquire); + std::lock_guard lock(mutex_); + return head_ - tail_; } size_t getHead() const { - return head_.load(std::memory_order_acquire); + std::lock_guard lock(mutex_); + return head_; } size_t getTail() const { - return tail_.load(std::memory_order_acquire); + std::lock_guard lock(mutex_); + return tail_; } bool empty() const { @@ -119,16 +166,244 @@ public: } void clear() { - head_.store(0, std::memory_order_release); - tail_.store(0, std::memory_order_release); + { + std::lock_guard lock(mutex_); + // Keep sequence numbers monotonic so cursors created before clear() + // cannot alias newly published slots after the reset. + tail_ = head_; + } + condition_.notify_all(); } private: - std::vector buffer_; + mutable std::vector buffer_; const size_t capacity_; + mutable std::mutex mutex_; + mutable std::condition_variable condition_; + size_t head_{0}; // 单调写序号；clear() 仅推进 tail_，避免旧游标 ABA。 + size_t tail_{0}; // 当前仍保留的最旧序号，同时也是 pop() 的共享读指针。 +}; - std::atomic head_; // 共享写指针 - std::atomic tail_; // 共享读指针（仅用于 SPSC 模式） +// 线程安全的多消费者广播缓冲区。缓冲区只保存不可变共享对象，消费者通过各自 +// 的 Cursor 独立前进；慢消费者被覆盖的数据会累计到 Cursor::dropped_count。 +// Cursor 是单线程所有权对象，不可由多个线程同时读写；每个消费者应创建自己的 Cursor。 +template +class BroadcastFrameRing { +public: + using ValuePtr = std::shared_ptr; + + enum class StartPosition { + NEXT_PUBLISHED, + OLDEST_AVAILABLE, + LATEST_AVAILABLE + }; + + struct Cursor { + uint64_t generation{0}; + uint64_t next_sequence{0}; + uint64_t dropped_count{0}; + StartPosition start_position{StartPosition::NEXT_PUBLISHED}; + + private: + bool generation_changed{false}; + uint64_t reported_dropped_count{0}; + friend class BroadcastFrameRing; + }; + + struct ReadResult { + ValuePtr value; + uint64_t generation{0}; + uint64_t sequence{0}; + uint64_t dropped_count{0}; + uint64_t dropped_since_last_read{0}; + bool generation_changed{false}; + }; + + struct Stats { + size_t capacity{0}; + size_t size{0}; + uint64_t generation{0}; + uint64_t next_sequence{0}; + uint64_t dropped_count{0}; + bool closed{false}; + }; + + explicit BroadcastFrameRing(const size_t capacity) + : capacity_(capacity) { + if (capacity_ == 0) { + throw std::invalid_argument("BroadcastFrameRing capacity must be greater than zero"); + } + } + + BroadcastFrameRing(const BroadcastFrameRing&) = delete; + BroadcastFrameRing& operator=(const BroadcastFrameRing&) = delete; + + Cursor makeCursor(const StartPosition start_position = StartPosition::NEXT_PUBLISHED) const { + std::lock_guard lock(mutex_); + Cursor cursor; + cursor.generation = generation_; + cursor.start_position = start_position; + cursor.next_sequence = startSequenceLocked_(start_position); + return cursor; + } + + std::optional publish(ValuePtr value) { + if (!value) { + return std::nullopt; + } + + std::optional published_sequence; + { + std::lock_guard lock(mutex_); + if (closed_) { + return std::nullopt; + } + + const uint64_t sequence = next_sequence_++; + if (entries_.size() == capacity_) { + entries_.pop_front(); + ++dropped_count_; + } + entries_.push_back(Entry{generation_, sequence, std::move(value)}); + published_sequence = sequence; + } + condition_.notify_all(); + return published_sequence; + } + + std::optional tryRead(Cursor& cursor) const { + std::lock_guard lock(mutex_); + return tryReadLocked_(cursor); + } + + template + std::optional waitRead( + Cursor& cursor, + const std::chrono::duration& timeout) const { + const auto deadline = std::chrono::steady_clock::now() + timeout; + std::unique_lock lock(mutex_); + while (true) { + if (auto result = tryReadLocked_(cursor)) { + return result; + } + if (closed_) { + return std::nullopt; + } + if (condition_.wait_until(lock, deadline) == std::cv_status::timeout) { + return tryReadLocked_(cursor); + } + } + } + + // 开始一个新的发布代次。旧 Cursor 会在下一次成功读取时收到 + // generation_changed=true，序号从 0 重新开始。 + uint64_t reset() { + uint64_t generation = 0; + { + std::lock_guard lock(mutex_); + entries_.clear(); + ++generation_; + next_sequence_ = 0; + closed_ = false; + generation = generation_; + } + condition_.notify_all(); + return generation; + } + + void close() { + { + std::lock_guard lock(mutex_); + closed_ = true; + } + condition_.notify_all(); + } + + bool closed() const { + std::lock_guard lock(mutex_); + return closed_; + } + + Stats stats() const { + std::lock_guard lock(mutex_); + return Stats{capacity_, entries_.size(), generation_, next_sequence_, dropped_count_, closed_}; + } + +private: + struct Entry { + uint64_t generation; + uint64_t sequence; + ValuePtr value; + }; + + uint64_t startSequenceLocked_(const StartPosition start_position) const { + if (entries_.empty()) { + return next_sequence_; + } + switch (start_position) { + case StartPosition::OLDEST_AVAILABLE: + return entries_.front().sequence; + case StartPosition::LATEST_AVAILABLE: + return entries_.back().sequence; + case StartPosition::NEXT_PUBLISHED: + default: + return next_sequence_; + } + } + + void synchronizeCursorGenerationLocked_(Cursor& cursor) const { + if (cursor.generation == generation_) { + return; + } + cursor.generation = generation_; + cursor.next_sequence = startSequenceLocked_(cursor.start_position); + cursor.generation_changed = true; + } + + std::optional tryReadLocked_(Cursor& cursor) const { + synchronizeCursorGenerationLocked_(cursor); + if (entries_.empty()) { + return std::nullopt; + } + + const uint64_t oldest_sequence = entries_.front().sequence; + if (cursor.next_sequence < oldest_sequence) { + cursor.dropped_count += oldest_sequence - cursor.next_sequence; + cursor.next_sequence = oldest_sequence; + } + if (cursor.next_sequence >= next_sequence_) { + return std::nullopt; + } + + const size_t index = static_cast(cursor.next_sequence - oldest_sequence); + if (index >= entries_.size()) { + return std::nullopt; + } + + const Entry& entry = entries_[index]; + ++cursor.next_sequence; + const uint64_t dropped_since_last = cursor.dropped_count - cursor.reported_dropped_count; + cursor.reported_dropped_count = cursor.dropped_count; + + ReadResult result; + result.value = entry.value; + result.generation = entry.generation; + result.sequence = entry.sequence; + result.dropped_count = cursor.dropped_count; + result.dropped_since_last_read = dropped_since_last; + result.generation_changed = cursor.generation_changed; + cursor.generation_changed = false; + return result; + } + + const size_t capacity_; + mutable std::mutex mutex_; + mutable std::condition_variable condition_; + std::deque entries_; + uint64_t generation_{1}; + uint64_t next_sequence_{0}; + uint64_t dropped_count_{0}; + bool closed_{false}; }; diff --git a/cmvr-es/common/media/media_frame.h b/cmvr-es/common/media/media_frame.h new file mode 100644 index 00000000..a5896b04 --- /dev/null +++ b/cmvr-es/common/media/media_frame.h @@ -0,0 +1,236 @@ +#ifndef CMVR_ES_COMMON_MEDIA_MEDIA_FRAME_H +#define CMVR_ES_COMMON_MEDIA_MEDIA_FRAME_H + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace cmvr::media { + +enum class MediaKind : uint8_t { + UNKNOWN = 0, + VIDEO = 1, + AUDIO = 2 +}; + +enum class Codec : uint8_t { + UNKNOWN = 0, + H264 = 1, + H265 = 2, + OPUS = 3, + PCM_S16LE = 4, + AAC = 5 +}; + +enum class PayloadFormat : uint8_t { + UNKNOWN = 0, + ANNEX_B = 1, + AVCC = 2, + RAW = 3, + OPUS_PACKET = 4, + AAC_ADTS = 5 +}; + +struct Rational { + int32_t numerator{0}; + int32_t denominator{1}; + + constexpr bool valid() const noexcept { + return numerator > 0 && denominator > 0; + } +}; + +constexpr bool operator==(const Rational lhs, const Rational rhs) noexcept { + return lhs.numerator == rhs.numerator && lhs.denominator == rhs.denominator; +} + +constexpr bool operator!=(const Rational lhs, const Rational rhs) noexcept { + return !(lhs == rhs); +} + +// A TrackDescriptor is immutable after construction. Reconfiguration is represented by +// publishing a new descriptor with a larger generation and attaching it to later frames. +class TrackDescriptor final { +public: + struct Config { + std::string id; + std::string source_id; + MediaKind kind{MediaKind::UNKNOWN}; + Codec codec{Codec::UNKNOWN}; + PayloadFormat payload_format{PayloadFormat::UNKNOWN}; + Rational time_base{}; + uint32_t width{0}; + uint32_t height{0}; + uint32_t sample_rate{0}; + uint32_t channels{0}; + uint32_t nominal_rate{0}; + float fx{0.0F}; + float fy{0.0F}; + float cx{0.0F}; + float cy{0.0F}; + std::vector distortion; + uint64_t generation{1}; + std::vector codec_config; + }; + + explicit TrackDescriptor(Config config) + : id(std::move(config.id)), + source_id(std::move(config.source_id)), + kind(config.kind), + codec(config.codec), + payload_format(config.payload_format), + time_base(config.time_base), + width(config.width), + height(config.height), + sample_rate(config.sample_rate), + channels(config.channels), + nominal_rate(config.nominal_rate), + fx(config.fx), + fy(config.fy), + cx(config.cx), + cy(config.cy), + distortion(std::move(config.distortion)), + generation(config.generation), + codec_config(std::move(config.codec_config)) { + if (id.empty()) { + throw std::invalid_argument("TrackDescriptor id must not be empty"); + } + if (source_id.empty()) { + throw std::invalid_argument("TrackDescriptor source_id must not be empty"); + } + if (kind == MediaKind::UNKNOWN) { + throw std::invalid_argument("TrackDescriptor kind must not be UNKNOWN"); + } + if (!time_base.valid()) { + throw std::invalid_argument("TrackDescriptor time_base is invalid"); + } + if (generation == 0) { + throw std::invalid_argument("TrackDescriptor generation must be greater than zero"); + } + } + + const std::string id; + const std::string source_id; + const MediaKind kind; + const Codec codec; + const PayloadFormat payload_format; + const Rational time_base; + const uint32_t width; + const uint32_t height; + const uint32_t sample_rate; + const uint32_t channels; + const uint32_t nominal_rate; + const float fx; + const float fy; + const float cx; + const float cy; + const std::vector distortion; + const uint64_t generation; + const std::vector codec_config; +}; + +using TrackDescriptorPtr = std::shared_ptr; + +inline bool equivalentTrackDescriptor( + const TrackDescriptor& lhs, + const TrackDescriptor& rhs) noexcept { + return lhs.id == rhs.id && + lhs.source_id == rhs.source_id && + lhs.kind == rhs.kind && + lhs.codec == rhs.codec && + lhs.payload_format == rhs.payload_format && + lhs.time_base == rhs.time_base && + lhs.width == rhs.width && + lhs.height == rhs.height && + lhs.sample_rate == rhs.sample_rate && + lhs.channels == rhs.channels && + lhs.nominal_rate == rhs.nominal_rate && + lhs.fx == rhs.fx && + lhs.fy == rhs.fy && + lhs.cx == rhs.cx && + lhs.cy == rhs.cy && + lhs.distortion == rhs.distortion && + lhs.generation == rhs.generation && + lhs.codec_config == rhs.codec_config; +} + +// MediaFrame and its payload are immutable and therefore safe to share across all protocol +// adapters and consumers without copying. PTS/DTS use TrackDescriptor::time_base; +// capture_time_ns is monotonic for pacing, while capture_utc_ns is optional wall time. +class MediaFrame final { +public: + using Payload = std::vector; + using PayloadPtr = std::shared_ptr; + + struct Config { + TrackDescriptorPtr descriptor; + Payload payload; + uint64_t sequence{0}; + int64_t pts{0}; + int64_t dts{0}; + int64_t duration{0}; + uint64_t capture_time_ns{0}; + int64_t capture_utc_ns{0}; + bool key_frame{false}; + bool discontinuity{false}; + }; + + explicit MediaFrame(Config config) + : descriptor(std::move(config.descriptor)), + payload(std::make_shared(std::move(config.payload))), + sequence(config.sequence), + pts(config.pts), + dts(config.dts), + duration(config.duration), + capture_time_ns(config.capture_time_ns), + capture_utc_ns(config.capture_utc_ns), + key_frame(config.key_frame), + discontinuity(config.discontinuity) { + if (!descriptor) { + throw std::invalid_argument("MediaFrame descriptor must not be null"); + } + } + + const uint8_t* data() const noexcept { + return payload->empty() ? nullptr : payload->data(); + } + + size_t size() const noexcept { + return payload->size(); + } + + bool empty() const noexcept { + return payload->empty(); + } + + const TrackDescriptorPtr descriptor; + const PayloadPtr payload; + const uint64_t sequence; + const int64_t pts; + const int64_t dts; + const int64_t duration; + const uint64_t capture_time_ns; + const int64_t capture_utc_ns; + const bool key_frame; + const bool discontinuity; +}; + +using MediaFramePtr = std::shared_ptr; + +inline TrackDescriptorPtr makeTrackDescriptor(TrackDescriptor::Config config) { + return std::make_shared(std::move(config)); +} + +inline MediaFramePtr makeMediaFrame(MediaFrame::Config config) { + return std::make_shared(std::move(config)); +} + +} // namespace cmvr::media + +#endif // CMVR_ES_COMMON_MEDIA_MEDIA_FRAME_H diff --git a/cmvr-es/config/README.md b/cmvr-es/config/README.md new file mode 100644 index 00000000..778d3c2f --- /dev/null +++ b/cmvr-es/config/README.md @@ -0,0 +1,112 @@ +# Config 模块开发指南 + +`config/` 保存 CMVR-ES 的默认运行配置。配置格式是 Protobuf TextFormat，Schema 位于 [`../../protos/cmvr/config/`](../../protos/cmvr/config/)。 + +返回[项目总览](../../README.md)。 + +## 配置树 + +```text +cmvr_es.pb.txt +├── logger/logger.pb.txt +├── manager/device_manager.pb.txt +│ └── devices//*.pb.txt +└── manager/task_manager.pb.txt + └── tasks//*.pb.txt +``` + +入口文件： + +- [`cmvr_es.pb.txt`](cmvr_es.pb.txt) +- [`manager/device_manager.pb.txt`](manager/device_manager.pb.txt) +- [`manager/task_manager.pb.txt`](manager/task_manager.pb.txt) + +## 路径规则 + +无参数运行时，程序读取： + +```text +/config/cmvr_es.pb.txt +``` + +安装后的 `output/bin/cmvr_es` 因此会读取 `output/bin/config/cmvr_es.pb.txt`；直接运行 `build/cmvr_es` 则会查找 `build/config/cmvr_es.pb.txt`，不会自动跳到安装目录。传入显式根配置时： + +```bash +./output/bin/cmvr_es /etc/cmvr-es/cmvr_es.pb.txt +``` + +设备、任务和证书等相对配置路径均以根配置文件所在目录解析。模型等资源通过 `ConfigHelper::resolveResourceFile()` 在配置根及父目录中查找；生产部署仍建议使用明确绝对路径。 + +日志配置中的相对 `directory` 以可执行文件目录解析，不以配置根解析。 + +## 新增设备配置 + +增加同类设备后端时： + +1. 在 `protos/cmvr/config/_config/` 增加后端 message； +2. 在类别设备 message 的 `oneof backend` 中增加字段； +3. 在 `devices//` 的 `.pb.txt` 中增加实例； +4. 实例外层 `id` 必须唯一； +5. 在 [`manager/device_manager.pb.txt`](manager/device_manager.pb.txt) 增加相同 `id`、正确 `type` 和配置路径； +6. 开发默认保持 `enable: false`； +7. 同步类别 factory 和 CMake； +8. 在无硬件环境验证关闭状态，在真机环境单独开启。 + +设备集合中的 ID 与 DeviceManager 条目 ID 不一致时，工厂会拒绝创建。 + +## 新增任务配置 + +1. 在 `protos/cmvr/config/` 增加任务配置和 root message； +2. 在 `tasks//` 增加默认 `.pb.txt`； +3. 在 `task_manager.pb.txt` 增加唯一任务 ID； +4. 配置正确的 `TaskType` 和 `TaskRunMode`； +5. 周期任务设置大于零的 `control_period_s`； +6. 服务任务使用 `TASK_RUN_MODE_BLOCKING_SERVICE`； +7. 默认关闭依赖网络、证书或硬件的新任务。 + +任务实现流程见 [`../task/README.md`](../task/README.md)。 + +## 默认值与校验 + +- 不依赖 proto3 数值零值表达危险的生产默认值； +- timeout、队列大小、帧大小和周期应在代码中校验； +- 新增 loader 对不认识的 enum 和未设置的 oneof 必须明确失败；当前个别历史路径仍有退化默认行为，不应复制； +- 设备端口、坐标系、速度和单位写入注释； +- `enable` 应由 manager 层控制，后端内部的 enable 字段不能替代 manager 开关； +- QUIC 需要 TaskManager 与 `QuicEdgeConfig.enable` 同时开启； +- QUIC 零媒体轨道是合法配置。 + +## 配置验证 + +构建后可以用 `protoc --encode` 对单个 TextFormat 文件做语法和字段验证。例如： + +```bash +output/bin/protoc \ + -I protos \ + --encode=cmvr.config.QuicEdgeRootConfig \ + protos/cmvr/config/quic_edge_config/quic_edge_config.proto \ + < cmvr-es/config/tasks/quic_edge_task/quic_edge_task.pb.txt \ + > /tmp/quic_edge_config.pb +``` + +该命令只验证 Proto Text 解析，不验证文件、设备、证书、网络和跨字段语义。最终仍需运行组件测试和进程烟雾测试。 + +## 生产配置 + +`cmake --install` 会重建 `output/bin/config/`。生产配置应复制到 `/etc/cmvr-es/` 等外部目录并显式传入。 + +- 不提交真实设备密码、token、私钥和生产地址； +- 证书与私钥放在独立 `certs/`，使用最小读取权限； +- 为不同站点维护独立配置根，不在运行时修改仓库样例； +- 发布前检查所有 `enable`、IP、端口和设备 ID； +- 变更配置 Schema 时同步 Proto 兼容性文档和平台生成代码。 + +## 提交检查 + +- [ ] TextFormat 可以被对应 root message 解析 +- [ ] ID、类别和引用路径完全一致 +- [ ] 新硬件和新网络任务默认关闭 +- [ ] 参数单位、范围和安全默认值明确 +- [ ] 没有生产凭据 +- [ ] 安装覆盖不会丢失现场配置 +- [ ] 无设备启动仍然成功 diff --git a/cmvr-es/config/manager/device_manager.pb.txt b/cmvr-es/config/manager/device_manager.pb.txt index 02bcd4db..46167ec9 100644 --- a/cmvr-es/config/manager/device_manager.pb.txt +++ b/cmvr-es/config/manager/device_manager.pb.txt @@ -70,26 +70,30 @@ device_manager { id: "hikvision_cam" type: DEVICE_TYPE_CAMERA config_file: "devices/camera/camera.pb.txt" - enable: true + # Host-development default: keep physical cameras disabled. + enable: false } devices { id: "hikvision_thermal_cam" type: DEVICE_TYPE_CAMERA config_file: "devices/camera/camera.pb.txt" - enable: true + # Host-development default: keep physical cameras disabled. + enable: false } devices { id: "mic1" type: DEVICE_TYPE_MICROPHONE config_file: "devices/microphone/microphone.pb.txt" - enable: true + # Host-development default: keep physical audio devices disabled. + enable: false } devices { id: "spk1" type: DEVICE_TYPE_SPEAKER config_file: "devices/speaker/speaker.pb.txt" - enable: true + # Host-development default: keep physical audio devices disabled. + enable: false } } diff --git a/cmvr-es/config/manager/task_manager.pb.txt b/cmvr-es/config/manager/task_manager.pb.txt index db13dfa9..3a749976 100644 --- a/cmvr-es/config/manager/task_manager.pb.txt +++ b/cmvr-es/config/manager/task_manager.pb.txt @@ -14,4 +14,12 @@ task_manager { config_file: "tasks/grpc_server_task/grpc_server_task.pb.txt" enable: true } + tasks { + id: "quic_edge" + type: TASK_TYPE_QUIC_EDGE + run_mode: TASK_RUN_MODE_BLOCKING_SERVICE + config_file: "tasks/quic_edge_task/quic_edge_task.pb.txt" + # Host-development default: no MsQuic package or physical media devices. + enable: false + } } diff --git a/cmvr-es/config/tasks/quic_edge_task/quic_edge_task.pb.txt b/cmvr-es/config/tasks/quic_edge_task/quic_edge_task.pb.txt new file mode 100644 index 00000000..6a958bad --- /dev/null +++ b/cmvr-es/config/tasks/quic_edge_task/quic_edge_task.pb.txt @@ -0,0 +1,65 @@ +quic_edge { + id: "quic_edge" + + # The repository does not currently ship MsQuic. Keep this disabled until + # the MsQuic dependency is installed and its backend is enabled at build time. + enable: false + + server_host: "quic-gateway.example.com" + server_port: 4433 + alpn: "cmvr-quic-edge/1" + node_id: "cmvr-edge" + software_version: "0.1" + + # The existing cmvr-es gRPC server remains the robot-control endpoint. "auto" + # selects a usable address from the interface snapshot sent at registration + # and on every heartbeat. + grpc_endpoint_host: "auto" + grpc_endpoint_port: 50052 + grpc_endpoint_tls: false + include_loopback_interfaces: false + + heartbeat_interval_ms: 5000 + control_response_timeout_ms: 1000 + + tls { + ca_file: "certs/quic_gateway_ca.pem" + certificate_file: "certs/cmvr_edge_cert.pem" + private_key_file: "certs/cmvr_edge_key.pem" + server_name: "quic-gateway.example.com" + allow_insecure: false + } + + reconnect { + initial_delay_ms: 500 + maximum_delay_ms: 30000 + multiplier: 2.0 + jitter_percent: 20 + connect_timeout_ms: 5000 + } + + maximum_datagram_bytes: 1200 + maximum_control_frame_bytes: 1048576 + # With 1200-byte DATAGRAMs and a 512-entry queue, 524288 stays below + # the atomic batch capacity while reserving slots for control messages. + maximum_frame_bytes: 524288 + datagram_send_queue_depth: 512 + media_poll_interval_ms: 2 + + # Zero media tracks is valid and keeps registration, IP reporting and + # heartbeat active. Add tracks only for devices enabled in DeviceManager. + # tracks { + # track_id: 1 + # source_kind: SOURCE_KIND_CAMERA + # device_id: "right_hand_cam" + # source_track_id: "right_hand_cam/video/color" + # enable: true + # } + # tracks { + # track_id: 2 + # source_kind: SOURCE_KIND_MICROPHONE + # device_id: "mic1" + # source_track_id: "mic1/audio/main" + # enable: true + # } +} diff --git a/cmvr-es/devices/README.md b/cmvr-es/devices/README.md new file mode 100644 index 00000000..646eaa18 --- /dev/null +++ b/cmvr-es/devices/README.md @@ -0,0 +1,355 @@ +# Devices 模块开发指南 + +`devices/` 屏蔽厂商 SDK、通信总线和硬件型号差异，对 manager、service、task 和算法层提供稳定的设备能力接口。 + +仓库真实目录名是 `cmvr-es/devices/`。设备层使用复数 `devices`，不是 `cmvr_es/device`。 + +返回[项目总览](../../README.md)。 + +## 创建链路 + +```text +config/cmvr_es.pb.txt + -> config/manager/device_manager.pb.txt + -> DeviceManager + -> DeviceFactory + 按 DeviceConfigEntry::DeviceType 选择设备大类 + -> CameraFactory / AGVFactory / RobotArmFactory / ... + 按设备配置 oneof backend 选择厂商实现 + -> 具体设备类 + -> AbstractDevice 和设备大类抽象接口 +``` + +关键文件： + +- [`abstract_device.h`](abstract_device.h) +- [`device_types.h`](device_types.h) +- [`../manager/device_manager/include/device_manager.h`](../manager/device_manager/include/device_manager.h) +- [`../manager/device_manager/src/device_factory.cpp`](../manager/device_manager/src/device_factory.cpp) +- [`../../protos/cmvr/config/device_manager_config/device_manager_config.proto`](../../protos/cmvr/config/device_manager_config/device_manager_config.proto) +- [`../config/manager/device_manager.pb.txt`](../config/manager/device_manager.pb.txt) + +协议层和业务任务不应直接依赖厂商 SDK 类型。厂商错误码、报文和连接细节由具体后端转换为抽象接口的通用语义。 + +## 当前可由配置创建的设备 + +| 大类 | 抽象接口 | 类别工厂 | 当前可选后端 | +| --- | --- | --- | --- | +| Camera | [`camera/abstract_camera.h`](camera/abstract_camera.h) | [`camera/camera_factory.h`](camera/camera_factory.h) | UVC、RealSense、Hikvision | +| AGV | [`agv/abstract_agv.h`](agv/abstract_agv.h) | [`agv/agv_factory.h`](agv/agv_factory.h) | MyAgv、SRC1100 | +| RobotArm | [`arm/robot_arm.h`](arm/robot_arm.h) | [`arm/robot_arm_factory.h`](arm/robot_arm_factory.h) | MotorRobotArm、AUBO、Huayan | +| DexHand | [`dexhand/abstract_dexhand.h`](dexhand/abstract_dexhand.h) | [`dexhand/dexhand_factory.h`](dexhand/dexhand_factory.h) | RH56DFTP、PX6AXGen3 | +| Microphone | [`microphone/abstract_microphone.h`](microphone/abstract_microphone.h) | [`microphone/microphone_factory.h`](microphone/microphone_factory.h) | FFmpeg | +| Speaker | [`speaker/abstract_speaker.h`](speaker/abstract_speaker.h) | [`speaker/speaker_factory.h`](speaker/speaker_factory.h) | FFmpeg | +| BioHead | [`biohead/abstract_biohead.h`](biohead/abstract_biohead.h) | DeviceFactory 直接创建 | BioHeadRobot | +| MotorSystem | `motor/motor_system/` | DeviceFactory 直接创建 | CAN/MuJoCo motor group | + +代码目录存在不等于已经接入配置创建链： + +- MechMind Proto 和实现仍存在，但当前 CameraFactory 明确拒绝创建； +- MujocoCamera 有实现并参与部分构建，但当前没有 CameraFactory 分支； +- Battery、Gripper、Robot、CanBus 等抽象或实现不一定已注册到 DeviceFactory； +- 所有已注册设备共用一个全局 ID 命名空间。 + +新增能力前先确认“已有代码”“可被 CMake 构建”“可被 Factory 创建”“可被 DeviceManager 配置启用”四个状态，不要混为一谈。 + +## 新增同类厂商后端 + +以新增 Camera 后端为例。 + +### 1. 扩展配置 Proto + +修改： + +```text +protos/cmvr/config/camera_config/camera_config.proto +``` + +新增厂商 config，并加入 `CameraDeviceConfig.oneof backend`。只使用新的字段 tag，不复用 reserved 或已发布 tag。 + +### 2. 新建后端目录 + +```text +camera/vendor_camera/ +├── CMakeLists.txt +├── include/ +│ └── vendor_camera.h +├── src/ +│ └── vendor_camera.cpp +└── tests/ + └── vendor_camera_test.cpp +``` + +### 3. 实现抽象接口 + +至少实现： + +- `typeName()`； +- `init()`； +- 实际支持的 `start()` / `stop()`； +- 状态查询和该类别核心能力； +- 若支持实时媒体，完整实现流接口和并发停止。 + +不要为了厂商特例向抽象类加入 SDK handle、私有报文或厂商专有结构。只有多个后端都需要的稳定语义才进入抽象接口或 `common/types/`。 + +### 4. 类别工厂注册 + +在 [`camera/camera_factory.h`](camera/camera_factory.h) 的 `backend_case()` 增加创建分支。 + +AGV、DexHand、Microphone、Speaker 等遵循相同模式。新增同类后端通常不需要修改全局 DeviceFactory。 + +### 5. CMake 聚合 + +1. 在 `camera/CMakeLists.txt` 增加 `add_subdirectory(vendor_camera)`； +2. 让类别 target 链接新后端 target； +3. 安装需要随应用分发的共享库； +4. 厂商 SDK 路径使用 `dependency/${ARCH}/third_party/...`，不能硬编码 x86； +5. 需要特殊 RPATH 时参考 Hikvision、Huayan 等现有实现。 + +推荐 target 形式： + +```cmake +add_library(vendor_camera SHARED + src/vendor_camera.cpp +) + +target_include_directories(vendor_camera + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} +) + +target_link_libraries(vendor_camera + PUBLIC cmvr_es::proto + PRIVATE vendor_sdk +) + +add_library(cmvr_es::device::vendor_camera ALIAS vendor_camera) +install(TARGETS vendor_camera LIBRARY DESTINATION lib) +``` + +### 6. 默认配置 + +在类别集合配置增加实例，并在 DeviceManager 配置增加同 ID 条目： + +```protobuf +devices { + id: "camera_front" + type: DEVICE_TYPE_CAMERA + config_file: "devices/camera/camera.pb.txt" + enable: false +} +``` + +新硬件默认 `enable: false`，确保无设备开发主机仍可启动。 + +## 新增全新设备大类 + +当现有抽象无法表达新设备能力时，还需要： + +1. 在 [`device_types.h`](device_types.h) 增加 `DeviceKind` 和 `toString()`； +2. 新增类别抽象接口； +3. 将稳定公共数据放入 `../common/types//`； +4. 新增配置 Proto； +5. 在 `DeviceConfigEntry::DeviceType` 使用新的 enum 数值； +6. 在 `DeviceFactory::DeviceFactory()` 注册 creator； +7. 更新 DeviceManager 的 type-to-string 日志映射； +8. 为 `getDevice()` 增加显式模板实例化； +9. 更新 `devices/CMakeLists.txt`、类别 CMake 和 device_manager target 链接； +10. 若平台需要访问，独立增加 API Proto 和 gRPC service。 + +不得复用 `device_manager_config.proto` 中已 reserved 的 enum 数值或名称。新增设备类别不会自动生成平台 RPC。 + +## 配置和 ID + +集合类型设备必须满足： + +```text +DeviceConfigEntry.id + == +CameraDeviceConfig / AGVDeviceConfig / ... 的外层 id +``` + +厂商 backend 内部 ID 可以为空，由类别工厂补齐；若填写，也必须与外层 ID 相同。 + +其他约束： + +- 相对配置路径以根 `cmvr_es.pb.txt` 所在目录解析； +- 生产密码、token 和证书不得提交到样例配置； +- 单个设备 init 失败时会被跳过，进程仍可能继续启动； +- 有初始化依赖的设备按配置顺序排列，例如 MotorSystem 在依赖它的 RobotArm 前； +- DeviceManager stop 遍历 unordered_map，不能依赖跨设备停止顺序； +- 当前不支持 service 运行期间并发热插拔设备集合。 + +配置细节见 [`../config/README.md`](../config/README.md)。 + +## 生命周期约定 + +| 接口 | 当前语义 | +| --- | --- | +| 构造函数 | 保存配置和轻量校验，不启动长期工作线程 | +| `init()` | DeviceManager 构造期间对启用设备调用 | +| `start()` | 当前 main 不统一调用，由 RPC、媒体或其他 owner 显式触发 | +| `startStreaming()` | 获取一个实时流生产租约 | +| `stopStreaming()` | 释放租约，最后一份租约停止生产者 | +| `stop()` | 完整停止设备，必须幂等 | +| `update()` | 当前没有统一 scheduler 自动调用 | +| 析构函数 | 回收线程、SDK callback、socket、fd 和 handle | + +实现要求： + +- init/start/stop 的重复调用有明确结果； +- stop 和最后一次 stopStreaming 返回前停止所有发布； +- 不持有 worker 退出所需的锁执行 join； +- SDK callback 不获取停止路径长期持有的控制锁； +- getState 使用与状态写入相同的锁； +- 含 `std::string`、vector 等状态不能无锁复制； +- 析构函数调用安全停止路径； +- callback 捕获对象前保证 owner 生命周期。 + +并发停止可参考： + +- [`camera/uvc_camera/`](camera/uvc_camera/) +- [`camera/hikvision_camera/`](camera/hikvision_camera/) +- [`camera/hikvision_camera/tests/`](camera/hikvision_camera/tests/) + +当前 main 收到退出信号只停止 TaskManager，没有显式调用 DeviceManager stop，设备析构仍必须可靠。 + +## 摄像头与麦克风实时流 + +设备实现抽象流接口后，由 [`../manager/media_source_hub/`](../manager/media_source_hub/) 适配给 gRPC 和 QUIC，不应在设备后端实现两套协议代码。 + +当前 Hub 轨道： + +```text +/video/color +/audio/main +``` + +当前 adapter 只把 `StreamFrameData.rgbFrame` 注册为彩色视频轨道；depthFrame 尚未注册为 Hub 深度轨道。 + +### Camera 完整编码帧 + +每个 access unit 应正确填写： + +- `rgbFrame` +- `codec`：H.264 或 H.265 +- `width`、`height`、`fps` +- `bKey` +- `stream_epoch` +- `sequence` +- `capture_monotonic_ns` +- `capture_utc_ns` +- `pts`、`dts` +- `time_base_num`、`time_base_den` +- `duration` +- `discontinuity` +- `codec_config_generation` +- `codec_config` + +H.264/H.265 后端必须识别关键帧，并尽量实现 `requestKeyFrame()`。 + +### Microphone 完整音频包 + +应正确填写： + +- `data` +- `sample_rate` +- `channels` +- `format` 和 `codec` +- `nb_samples` +- sequence、时间戳、time base、duration +- stream epoch、discontinuity 和 codec generation + +### 生产者重启 + +采集/编码生产者真正停止并重新启动时： + +1. 增加 `stream_epoch`； +2. 将 source sequence 重置为 0； +3. 增加 `codec_config_generation`； +4. 确保后续帧携带完整的新编码元数据； +5. 将第一帧标记为 discontinuity； +6. 视频从关键帧恢复输出。 + +设备后端不直接发布 `TrackDescriptor`。`device_media_source_adapter.cpp` 会根据帧元数据生成或更新 descriptor。 + +### 运行期编码配置变化 + +编码器没有重启、只在同一 stream epoch 内改变分辨率、codec config 等参数时： + +1. 保持 `stream_epoch` 不变； +2. 保持 sequence 连续递增； +3. 增加 `codec_config_generation`，或让其他描述字段反映变化； +4. 在后续帧中携带新元数据； +5. 标记 discontinuity； +6. H.264/H.265 从新的关键帧恢复。 + +adapter 检测到描述变化后创建新 descriptor，设备后端不要自行维护协议侧 descriptor 状态。 + +## 设备侧环形队列安全 + +现有设备后端多使用 `SPMCRingBuffer`： + +- 只允许一个逻辑生产者； +- 每个消费者独立持有读游标； +- 同一读游标不能跨线程并发访问； +- 不对同一路读取混用无参 `pop()` 和带游标 `pop(index)`； +- 需要最新帧时使用 `getLatest(index)`； +- 不要先取 head 再分两步读取，避免检查/读取竞态； +- 满队列覆盖旧数据是实时媒体的预期行为； +- `waitEncodedFrame()` 必须有有限 timeout，不能永久阻塞。 + +MediaSourceHub Subscription 同样是单消费者对象，不同协议或客户端必须各自订阅。 + +发布后的 `MediaFrame`、`TrackDescriptor` 和 payload 不可再修改。 + +## 测试要求 + +至少覆盖： + +- 配置缺失、非法参数和 ID 不一致； +- Factory 选择正确后端； +- init/start/stop 重复执行； +- init 失败后无残留线程和句柄； +- stop 与 SDK callback 并发； +- 最后一个流租约释放后不再发布； +- 多消费者使用独立游标； +- 环形队列覆盖和丢帧； +- epoch、sequence、关键帧和 codec generation； +- 设备断开、超时和重连。 + +无硬件参考测试： + +- [`camera/hikvision_camera/tests/hikvision_camera_callback_test.cpp`](camera/hikvision_camera/tests/hikvision_camera_callback_test.cpp) +- [`../manager/media_source_hub/tests/media_source_hub_test.cpp`](../manager/media_source_hub/tests/media_source_hub_test.cpp) + +```bash +cmake -S . -B build \ + -DCMVR_ARCH=x86 \ + -DBUILD_TESTING=ON \ + -DCMVR_MEDIA_SOURCE_HUB_BUILD_TESTS=ON + +cmake --build build -j"$(nproc)" + +ctest \ + --test-dir build \ + -R 'hikvision_camera_callback_test|media_source_hub_test' \ + --output-on-failure +``` + +新测试必须在 `BUILD_TESTING` 下使用 `add_test()` 登记。只创建 executable 不会自动被 CTest 执行。 + +## 提交检查 + +- [ ] 抽象接口没有厂商 SDK 类型 +- [ ] manager、外层配置和 backend ID 一致 +- [ ] 新硬件配置默认关闭 +- [ ] 生命周期支持重复调用 +- [ ] stop 是同步发布屏障 +- [ ] 状态读写使用同一把锁 +- [ ] 每个实时流只有一个生产者 +- [ ] 每个消费者使用独立游标 +- [ ] 时间戳、sequence 和编码元数据完整 +- [ ] Factory 和类别 CMake 均已接入 +- [ ] 厂商运行库有安装规则 +- [ ] 有无真实硬件的自动测试 +- [ ] 平台能力变化已评估 Proto 和 service diff --git a/cmvr-es/devices/camera/abstract_camera.h b/cmvr-es/devices/camera/abstract_camera.h index d18898ad..8b5b8a36 100644 --- a/cmvr-es/devices/camera/abstract_camera.h +++ b/cmvr-es/devices/camera/abstract_camera.h @@ -2,6 +2,11 @@ #define CMVR_ES_ABSTRACT_CAMERA_H #pragma once +#include +#include +#include +#include + #include #include "../abstract_device.h" #include @@ -29,12 +34,27 @@ namespace cmvr::device { std::vector depthFrame; //编码格式 std::string codec = ".h264"; - Rs2Intrinsics intrinsics; - int width; - int height; - int fps; - bool bKey; - bool depthKey; + Rs2Intrinsics intrinsics{}; + int width = 0; + int height = 0; + int fps = 0; + bool bKey = false; + bool depthKey = false; + + // Protocol-neutral real-time metadata. The producer fills these values + // when a complete encoded access unit is published. + uint64_t stream_epoch = 0; + uint64_t sequence = 0; + int64_t capture_monotonic_ns = 0; + int64_t capture_utc_ns = 0; + int64_t pts = 0; + int64_t dts = 0; + int32_t time_base_num = 1; + int32_t time_base_den = 1; + int64_t duration = 0; + bool discontinuity = false; + uint32_t codec_config_generation = 0; + std::vector codec_config; }; enum class PtzCommand { @@ -64,7 +84,10 @@ namespace cmvr::device { ~AbstractCamera() override = default; DeviceKind kind() const noexcept override { return DeviceKind::Camera; } - inline void getState(CameraState &state) {state = state_;} + // Every implementation must take the same lock used by its state_ + // writers; CameraState contains std::string and cannot be snapshotted + // safely while another thread mutates it. + virtual void getState(CameraState &state) = 0; virtual void getRGBImage(cv::Mat &color, Rs2Intrinsics& intrinsics) {} virtual void getDepthImage(cv::Mat &depth, Rs2Intrinsics& intrinsics) {} virtual void getRGBDImages(cv::Mat &color, cv::Mat &depth, Rs2Intrinsics& intrinsics) {} @@ -73,6 +96,14 @@ namespace cmvr::device { virtual void pauseRecording() {} virtual void resumeRecording() {} virtual void getEncodedFrame(StreamFrameData& frame_data, size_t& index) {} + virtual bool waitEncodedFrame( + StreamFrameData& frame_data, + size_t& index, + std::chrono::milliseconds timeout) { + (void)timeout; + getEncodedFrame(frame_data, index); + return !frame_data.rgbFrame.empty() || !frame_data.depthFrame.empty(); + } virtual bool getLatestEncodedFrame(StreamFrameData& frame_data, size_t& next_index) { return false; } @@ -80,10 +111,12 @@ namespace cmvr::device { virtual bool startStreaming() {return true;} virtual void stopStreaming() {} virtual bool controlPtz(PtzCommand command, bool stop, int speed) { - state_.is_error = true; - state_.error_message = "PTZ control unsupported"; + (void)command; + (void)stop; + (void)speed; return false; } + virtual bool requestKeyFrame() { return false; } virtual Eigen::Vector3f get3DPointFromPixel(int u, int v) {return {0,0,0};} protected: CameraState state_{}; diff --git a/cmvr-es/devices/camera/hikvision_camera/CMakeLists.txt b/cmvr-es/devices/camera/hikvision_camera/CMakeLists.txt index 9308507b..432a1cb8 100644 --- a/cmvr-es/devices/camera/hikvision_camera/CMakeLists.txt +++ b/cmvr-es/devices/camera/hikvision_camera/CMakeLists.txt @@ -50,6 +50,9 @@ endif() find_path(HIKVISION_JSONCPP_INCLUDE_DIR NAMES json/json.h PATH_SUFFIXES jsoncpp) message(STATUS "[hikvision_camera] jsoncpp include dir: ${HIKVISION_JSONCPP_INCLUDE_DIR}") +if(HIKVISION_JSONCPP_INCLUDE_DIR) + target_include_directories(hikvision_camera PRIVATE "${HIKVISION_JSONCPP_INCLUDE_DIR}") +endif() target_link_libraries(hikvision_camera PUBLIC @@ -71,3 +74,49 @@ add_library(cmvr_es::device::hikvision_camera ALIAS hikvision_camera) install(TARGETS hikvision_camera LIBRARY DESTINATION lib) install(FILES ${HIKVISION_SDK_LIBS} DESTINATION lib) install(DIRECTORY "${HIKVISION_SDK_COM_DIR}" DESTINATION lib) + +if(BUILD_TESTING) + add_executable(hikvision_camera_callback_test + tests/hikvision_camera_callback_test.cpp + src/hikvision_camera.cpp + ) + target_include_directories(hikvision_camera_callback_test + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + "${HIKVISION_SDK_ROOT}/include" + ) + if(HIKVISION_JSONCPP_INCLUDE_DIR) + target_include_directories( + hikvision_camera_callback_test + PRIVATE + "${HIKVISION_JSONCPP_INCLUDE_DIR}" + ) + endif() + target_compile_definitions(hikvision_camera_callback_test + PRIVATE + HIKVISION_SDK_LIB_DIR="${HIKVISION_SDK_LIB_DIR}/" + ) + # The test supplies a small in-process HCNetSDK fake, so it exercises the + # callback/stop ordering without requiring a camera or loading hcnetsdk. + target_link_libraries(hikvision_camera_callback_test + PRIVATE + glog + opencv_core + cmvr_es::proto + ${HIKVISION_JSONCPP_LINK} + pthread + ) + add_test( + NAME hikvision_camera_callback_test + COMMAND hikvision_camera_callback_test + ) + set_tests_properties(hikvision_camera_callback_test PROPERTIES TIMEOUT 10) + if(UNIX AND NOT APPLE) + get_property(_hikvision_test_library_dirs DIRECTORY PROPERTY LINK_DIRECTORIES) + list(PREPEND _hikvision_test_library_dirs "${CMAKE_BINARY_DIR}/cmvr_compiler_runtime") + list(JOIN _hikvision_test_library_dirs ":" _hikvision_test_library_path) + set_tests_properties(hikvision_camera_callback_test PROPERTIES + ENVIRONMENT "LD_LIBRARY_PATH=${_hikvision_test_library_path}" + ) + endif() +endif() diff --git a/cmvr-es/devices/camera/hikvision_camera/include/hikvision_camera.h b/cmvr-es/devices/camera/hikvision_camera/include/hikvision_camera.h index 0e544ab0..ff438cd6 100644 --- a/cmvr-es/devices/camera/hikvision_camera/include/hikvision_camera.h +++ b/cmvr-es/devices/camera/hikvision_camera/include/hikvision_camera.h @@ -1,11 +1,10 @@ #ifndef CMVR_ES_HIKVISION_CAMERA_H #define CMVR_ES_HIKVISION_CAMERA_H -#include +#include #include #include #include -#include #include #include "common/base/ring_buffer.h" @@ -22,6 +21,7 @@ public: bool init() override; bool start() override; bool stop() override; + void getState(CameraState& state) override; void getRGBImage(cv::Mat& color, Rs2Intrinsics& intrinsics) override; void getDepthImage(cv::Mat& depth, Rs2Intrinsics& intrinsics) override; @@ -32,13 +32,16 @@ public: void pauseRecording() override; void resumeRecording() override; void getEncodedFrame(StreamFrameData& frame_data, size_t& index) override; + bool waitEncodedFrame(StreamFrameData& frame_data, size_t& index, std::chrono::milliseconds timeout) override; bool getLatestEncodedFrame(StreamFrameData& frame_data, size_t& next_index) override; bool startStreaming() override; void stopStreaming() override; bool controlPtz(PtzCommand command, bool stop, int speed) override; bool executeJsonCommand(const std::string& request_json, std::string& response_json) override; + bool requestKeyFrame() override; - void onEsData(unsigned int packet_type, + void onEsData(long real_handle, + unsigned int packet_type, unsigned char* buffer, unsigned int buffer_size, unsigned int width, @@ -86,11 +89,19 @@ private: std::string current_video_path_; mutable std::mutex ctrl_mtx_; - std::mutex stream_mtx_; + // SDK callbacks must never take ctrl_mtx_: NET_DVR_StopRealPlay may wait + // for an in-flight callback while stop() owns that mutex. This mutex is the + // single synchronization domain for callback publication and stream state. + mutable std::mutex callback_mtx_; + bool callback_publishing_enabled_ = false; + long callback_preview_handle_ = -1; std::vector es_stream_header_; bool has_es_stream_header_ = false; - std::atomic awaiting_key_frame_{false}; + bool awaiting_key_frame_ = false; int stream_count_ = 0; + uint64_t stream_epoch_ = 0; + uint64_t stream_sequence_ = 0; + uint32_t codec_config_generation_ = 0; }; } // namespace cmvr::device diff --git a/cmvr-es/devices/camera/hikvision_camera/src/hikvision_camera.cpp b/cmvr-es/devices/camera/hikvision_camera/src/hikvision_camera.cpp index b6255299..1645a602 100644 --- a/cmvr-es/devices/camera/hikvision_camera/src/hikvision_camera.cpp +++ b/cmvr-es/devices/camera/hikvision_camera/src/hikvision_camera.cpp @@ -1,8 +1,9 @@ #include "../include/hikvision_camera.h" -#include #include +#include #include +#include #include #include #include @@ -217,13 +218,14 @@ std::string makeJsonResult(bool success, const std::string& error_message = "") } void CALLBACK hikvisionEsRealPlayCallback( - LONG, NET_DVR_PACKET_INFO_EX* packet_info, void* user) + LONG real_handle, NET_DVR_PACKET_INFO_EX* packet_info, void* user) { auto* camera = static_cast(user); if (!camera || !packet_info) { return; } - camera->onEsData(packet_info->dwPacketType, + camera->onEsData(real_handle, + packet_info->dwPacketType, packet_info->pPacketBuffer, packet_info->dwPacketSize, packet_info->wWidth, @@ -252,6 +254,9 @@ HikvisionCamera::HikvisionCamera(const config::HikvisionCameraConfig& camera) codec_ = camera_.codec().empty() ? "H264" : camera_.codec(); sdk_path_ = normalizeSdkPath( camera_.sdk_path().empty() ? defaultRuntimeSdkPath() : camera_.sdk_path()); + // Keep the shared_ptr itself immutable after construction. Readers and the + // SDK callback may use it concurrently; the ring buffer owns its locking. + stream_frame_buffer_ = std::make_shared>(buffer_size_); state_.fps = fps_; state_.width = width_; @@ -282,7 +287,6 @@ bool HikvisionCamera::init() return false; } - stream_frame_buffer_ = std::make_shared>(buffer_size_); stream_frame_buffer_->clear(); resetStreamState_(); @@ -334,7 +338,6 @@ bool HikvisionCamera::stop() stopRecordingUnlocked_(); } state_.is_streaming = false; - awaiting_key_frame_.store(false, std::memory_order_release); stream_count_ = 0; resetStreamState_(); stopPreview_(); @@ -347,8 +350,15 @@ bool HikvisionCamera::stop() return true; } +void HikvisionCamera::getState(CameraState& state) +{ + std::lock_guard lock(ctrl_mtx_); + state = state_; +} + void HikvisionCamera::getRGBImage(cv::Mat& color, Rs2Intrinsics& intrinsics) { + std::lock_guard lock(ctrl_mtx_); clear_error_(); fillIntrinsics_(intrinsics); setError_("getRGBImage unsupported: HikvisionCamera only forwards stream data"); @@ -357,6 +367,7 @@ void HikvisionCamera::getRGBImage(cv::Mat& color, Rs2Intrinsics& intrinsics) void HikvisionCamera::getDepthImage(cv::Mat& depth, Rs2Intrinsics&) { + std::lock_guard lock(ctrl_mtx_); setError_("getDepthImage unsupported usage"); depth.release(); } @@ -418,17 +429,41 @@ void HikvisionCamera::stopRecordingUnlocked_() void HikvisionCamera::pauseRecording() { + std::lock_guard lock(ctrl_mtx_); setError_("pauseRecording unsupported by Hikvision SDK recording"); } void HikvisionCamera::resumeRecording() { + std::lock_guard lock(ctrl_mtx_); setError_("resumeRecording unsupported by Hikvision SDK recording"); } void HikvisionCamera::getEncodedFrame(StreamFrameData& frame_data, size_t& index) { - getLatestEncodedFrame(frame_data, index); + if (!stream_frame_buffer_) { + return; + } + auto frame = stream_frame_buffer_->pop(index); + if (frame) { + frame_data = std::move(*frame); + } +} + +bool HikvisionCamera::waitEncodedFrame( + StreamFrameData& frame_data, + size_t& index, + const std::chrono::milliseconds timeout) +{ + if (!stream_frame_buffer_) { + return false; + } + auto frame = stream_frame_buffer_->waitPop(index, timeout); + if (!frame) { + return false; + } + frame_data = std::move(*frame); + return !frame_data.rgbFrame.empty() || !frame_data.depthFrame.empty(); } bool HikvisionCamera::getLatestEncodedFrame(StreamFrameData& frame_data, size_t& next_index) @@ -437,19 +472,12 @@ bool HikvisionCamera::getLatestEncodedFrame(StreamFrameData& frame_data, size_t& return false; } - const size_t head = stream_frame_buffer_->getHead(); - if (head == 0) { - return false; - } - - size_t latest_index = head - 1; - auto frame = stream_frame_buffer_->pop(latest_index); + auto frame = stream_frame_buffer_->getLatest(next_index); if (!frame.has_value()) { return false; } frame_data = frame.value(); - next_index = latest_index; return true; } @@ -468,13 +496,25 @@ bool HikvisionCamera::startStreaming() if (stream_frame_buffer_) { stream_frame_buffer_->clear(); } - awaiting_key_frame_.store(true, std::memory_order_release); + std::lock_guard callback_lock(callback_mtx_); + if (callback_preview_handle_ < 0 || + callback_preview_handle_ != static_cast(real_handle_)) { + setError_("preview callback is not active"); + return false; + } + ++stream_epoch_; + stream_sequence_ = 0; + ++codec_config_generation_; + awaiting_key_frame_ = true; + callback_publishing_enabled_ = true; } - state_.is_streaming = true; ++stream_count_; if (first_stream && !requestKeyFrame_()) { - awaiting_key_frame_.store(false, std::memory_order_release); + // Keep waiting for the next natural I-frame. Publishing P/B frames + // immediately would make a newly attached decoder start corrupted. + CMVR_LOG(WARNING) << "[HikvisionCamera] key-frame request failed; " + << "waiting for the next natural I-frame"; } return true; } @@ -486,8 +526,13 @@ void HikvisionCamera::stopStreaming() --stream_count_; } if (stream_count_ == 0) { + // Wait for an already-running callback to finish its publication, then + // prevent both queued and future callbacks from publishing. No frame + // can be pushed after this critical section has completed. + std::lock_guard callback_lock(callback_mtx_); + callback_publishing_enabled_ = false; + awaiting_key_frame_ = false; state_.is_streaming = false; - awaiting_key_frame_.store(false, std::memory_order_release); } } @@ -517,6 +562,22 @@ bool HikvisionCamera::controlPtz(PtzCommand command, bool stop, int speed) return true; } +bool HikvisionCamera::requestKeyFrame() +{ + std::lock_guard lock(ctrl_mtx_); + clear_error_(); + if (!state_.is_opened || user_id_ < 0) { + setError_("camera not opened"); + return false; + } + if (!requestKeyFrame_()) { + setError_(sdkError_( + stream_type_ == 0 ? "NET_DVR_MakeKeyFrame" : "NET_DVR_MakeKeyFrameSub")); + return false; + } + return true; +} + bool HikvisionCamera::requestKeyFrame_() { if (user_id_ < 0) { @@ -524,7 +585,7 @@ bool HikvisionCamera::requestKeyFrame_() return false; } - const bool is_sub_stream = stream_type_ == 1; + const bool is_sub_stream = stream_type_ != 0; const BOOL success = is_sub_stream ? NET_DVR_MakeKeyFrameSub(user_id_, channel_) : NET_DVR_MakeKeyFrame(user_id_, channel_); @@ -579,7 +640,11 @@ bool HikvisionCamera::executeJsonCommand(const std::string& request_json, std::s const int speed = jsonIntField(root, "speed", 0); if (!controlPtz(ptz_command, stop, speed)) { - response_json = makeJsonResult(false, state_.error_message); + CameraState state; + getState(state); + response_json = makeJsonResult( + false, + state.error_message.empty() ? "failed to control PTZ" : state.error_message); return false; } @@ -595,21 +660,37 @@ bool HikvisionCamera::executeJsonCommand(const std::string& request_json, std::s } void HikvisionCamera::onEsData( - unsigned int packet_type, + const long real_handle, + const unsigned int packet_type, unsigned char* buffer, - unsigned int buffer_size, - unsigned int packet_width, - unsigned int packet_height) + const unsigned int buffer_size, + const unsigned int packet_width, + const unsigned int packet_height) { if (!buffer || buffer_size == 0) { return; } + // Never take ctrl_mtx_ from an SDK callback. NET_DVR_StopRealPlay may wait + // for this callback while stop() owns ctrl_mtx_. Holding callback_mtx_ + // through publication also preserves the ring's single-producer contract. + std::lock_guard callback_lock(callback_mtx_); + if (callback_preview_handle_ != real_handle || !stream_frame_buffer_) { + return; + } + if (packet_type == kHikvisionPacketFileHeader) { - std::lock_guard lock(stream_mtx_); - if (!has_es_stream_header_) { + const bool changed = + !has_es_stream_header_ || + es_stream_header_.size() != buffer_size || + !std::equal(es_stream_header_.begin(), es_stream_header_.end(), buffer); + if (changed) { es_stream_header_.assign(buffer, buffer + buffer_size); has_es_stream_header_ = true; + ++codec_config_generation_; + if (callback_publishing_enabled_) { + awaiting_key_frame_ = true; + } CMVR_LOG(INFO) << "[HikvisionCamera] received ES stream header, size=" << buffer_size; } return; @@ -634,16 +715,16 @@ void HikvisionCamera::onEsData( << packet_type << ", size=" << buffer_size; } - if (!state_.is_streaming || !stream_frame_buffer_) { + if (!callback_publishing_enabled_) { return; } const bool is_key_frame = packet_type == kHikvisionPacketVideoIFrame; - if (awaiting_key_frame_.load(std::memory_order_acquire)) { + if (awaiting_key_frame_) { if (!is_key_frame) { return; } - awaiting_key_frame_.store(false, std::memory_order_release); + awaiting_key_frame_ = false; CMVR_LOG(INFO) << "[HikvisionCamera] received first key frame after stream start"; } @@ -666,19 +747,39 @@ void HikvisionCamera::pushEncodedFrame_( } StreamFrameData frame_data; + const auto capture_monotonic = std::chrono::steady_clock::now(); + const auto capture_utc = std::chrono::system_clock::now(); frame_data.rgbFrame.assign(buffer, buffer + buffer_size); frame_data.codec = codec_; frame_data.fps = fps_; frame_data.width = packet_width > 0 ? static_cast(packet_width) : width_; frame_data.height = packet_height > 0 ? static_cast(packet_height) : height_; frame_data.bKey = is_key_frame; + frame_data.stream_epoch = stream_epoch_; + frame_data.sequence = stream_sequence_++; + frame_data.capture_monotonic_ns = std::chrono::duration_cast( + capture_monotonic.time_since_epoch()).count(); + frame_data.capture_utc_ns = std::chrono::duration_cast( + capture_utc.time_since_epoch()).count(); + frame_data.pts = static_cast(frame_data.sequence); + frame_data.dts = frame_data.pts; + frame_data.time_base_num = 1; + frame_data.time_base_den = std::max(1, fps_); + frame_data.duration = 1; + frame_data.codec_config_generation = codec_config_generation_; + // Hikvision labels packet type 0 as a "file header", but the bundled SDK + // does not guarantee that it is a decoder-ready VPS/SPS/PPS blob. Keep it + // only for change detection until its format is verified on real hardware. fillIntrinsics_(frame_data.intrinsics); stream_frame_buffer_->push(frame_data); } void HikvisionCamera::resetStreamState_() { - std::lock_guard lock(stream_mtx_); + std::lock_guard callback_lock(callback_mtx_); + callback_publishing_enabled_ = false; + callback_preview_handle_ = -1; + awaiting_key_frame_ = false; es_stream_header_.clear(); has_es_stream_header_ = false; } @@ -758,8 +859,20 @@ bool HikvisionCamera::startPreview_() setError_(sdkError_("NET_DVR_RealPlay_V40")); return false; } + { + std::lock_guard callback_lock(callback_mtx_); + callback_publishing_enabled_ = false; + callback_preview_handle_ = static_cast(real_handle_); + awaiting_key_frame_ = false; + } if (!NET_DVR_SetESRealPlayCallBack(real_handle_, hikvisionEsRealPlayCallback, this)) { setError_(sdkError_("NET_DVR_SetESRealPlayCallBack")); + { + std::lock_guard callback_lock(callback_mtx_); + callback_publishing_enabled_ = false; + callback_preview_handle_ = -1; + awaiting_key_frame_ = false; + } NET_DVR_StopRealPlay(real_handle_); real_handle_ = -1; return false; @@ -769,8 +882,18 @@ bool HikvisionCamera::startPreview_() void HikvisionCamera::stopPreview_() { - if (real_handle_ >= 0) { - NET_DVR_StopRealPlay(real_handle_); + const int preview_handle = real_handle_; + { + // Invalidate before asking the SDK to stop. We intentionally release + // callback_mtx_ before NET_DVR_StopRealPlay because that function may + // wait for an SDK callback to return. + std::lock_guard callback_lock(callback_mtx_); + callback_publishing_enabled_ = false; + callback_preview_handle_ = -1; + awaiting_key_frame_ = false; + } + if (preview_handle >= 0) { + NET_DVR_StopRealPlay(preview_handle); real_handle_ = -1; } } diff --git a/cmvr-es/devices/camera/hikvision_camera/tests/hikvision_camera_callback_test.cpp b/cmvr-es/devices/camera/hikvision_camera/tests/hikvision_camera_callback_test.cpp new file mode 100644 index 00000000..07ba6524 --- /dev/null +++ b/cmvr-es/devices/camera/hikvision_camera/tests/hikvision_camera_callback_test.cpp @@ -0,0 +1,382 @@ +#include "../include/hikvision_camera.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "HCNetSDK.h" + +namespace { + +using EsDataCallback = + void(CALLBACK*)(LONG, NET_DVR_PACKET_INFO_EX*, void*); + +constexpr DWORD kFileHeader = 0; +constexpr DWORD kVideoIFrame = 1; +constexpr DWORD kVideoPFrame = 3; + +std::mutex g_fake_sdk_mutex; +EsDataCallback g_es_data_callback = nullptr; +void* g_es_data_user = nullptr; +LONG g_real_handle = 42; +std::atomic g_stop_callback_count{0}; +std::atomic g_key_frame_request_count{0}; +std::atomic g_ptz_call_count{0}; +std::atomic g_last_ptz_command{0}; +std::atomic g_last_ptz_stop{0}; +std::atomic g_last_ptz_speed{0}; + +void resetFakeSdk() +{ + std::lock_guard lock(g_fake_sdk_mutex); + g_es_data_callback = nullptr; + g_es_data_user = nullptr; + g_stop_callback_count = 0; + g_key_frame_request_count = 0; + g_ptz_call_count = 0; + g_last_ptz_command = 0; + g_last_ptz_stop = 0; + g_last_ptz_speed = 0; +} + +void emitEsPacket( + const DWORD packet_type, + const LONG real_handle, + std::vector payload, + const WORD width = 640, + const WORD height = 360) +{ + EsDataCallback callback = nullptr; + void* user = nullptr; + { + std::lock_guard lock(g_fake_sdk_mutex); + callback = g_es_data_callback; + user = g_es_data_user; + } + + if (!callback) { + return; + } + + NET_DVR_PACKET_INFO_EX packet{}; + packet.wWidth = width; + packet.wHeight = height; + packet.dwPacketType = packet_type; + packet.dwPacketSize = static_cast(payload.size()); + packet.pPacketBuffer = payload.data(); + callback(real_handle, &packet, user); +} + +void emitIFrame(const LONG real_handle = g_real_handle) +{ + emitEsPacket( + kVideoIFrame, + real_handle, + {0x00, 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0x21}); +} + +void emitPFrame(const LONG real_handle = g_real_handle) +{ + emitEsPacket( + kVideoPFrame, + real_handle, + {0x00, 0x00, 0x00, 0x01, 0x41, 0x9A, 0x20}); +} + +bool check(const bool condition, const char* expression, const int line) +{ + if (condition) { + return true; + } + std::cerr << "CHECK failed at line " << line << ": " << expression << '\n'; + return false; +} + +#define CHECK_TRUE(expression) \ + do { \ + if (!check(static_cast(expression), #expression, __LINE__)) { \ + return false; \ + } \ + } while (false) + +cmvr::config::HikvisionCameraConfig makeConfig() +{ + cmvr::config::HikvisionCameraConfig config; + config.set_id("hikvision_callback_test"); + config.set_ip("127.0.0.1"); + config.set_username("admin"); + config.set_password("test-only"); + config.set_port(8000); + config.set_channel(1); + config.set_stream_type(0); + config.set_link_mode(0); + config.set_width(1920); + config.set_height(1080); + config.set_fps(25); + config.set_codec("H264"); + config.set_buffer_size(32); + return config; +} + +bool testCallbackPublicationLifecycle() +{ + resetFakeSdk(); + cmvr::device::HikvisionCamera camera(makeConfig()); + CHECK_TRUE(camera.init()); + CHECK_TRUE(camera.start()); + CHECK_TRUE(camera.startStreaming()); + CHECK_TRUE(g_key_frame_request_count.load() == 1); + + size_t cursor = 0; + cmvr::device::StreamFrameData frame; + + // A new subscriber must not receive an undecodable inter frame, and a + // delayed callback from an older preview handle must also be rejected. + emitPFrame(); + emitIFrame(g_real_handle - 1); + CHECK_TRUE(!camera.waitEncodedFrame( + frame, cursor, std::chrono::milliseconds(10))); + + emitIFrame(); + CHECK_TRUE(camera.waitEncodedFrame( + frame, cursor, std::chrono::milliseconds(50))); + CHECK_TRUE(frame.stream_epoch == 1); + CHECK_TRUE(frame.sequence == 0); + CHECK_TRUE(frame.codec_config_generation == 1); + CHECK_TRUE(frame.bKey); + CHECK_TRUE(frame.width == 640); + CHECK_TRUE(frame.height == 360); + + CHECK_TRUE(camera.requestKeyFrame()); + CHECK_TRUE(g_key_frame_request_count.load() == 2); + + CHECK_TRUE(camera.controlPtz( + cmvr::device::PtzCommand::PanLeft, false, 99)); + CHECK_TRUE(g_ptz_call_count.load() == 1); + CHECK_TRUE(g_last_ptz_command.load() == PAN_LEFT); + CHECK_TRUE(g_last_ptz_stop.load() == 0); + CHECK_TRUE(g_last_ptz_speed.load() == 7); + + std::string json_response; + CHECK_TRUE(camera.executeJsonCommand( + R"({"command":"ptz","direction":"zoom_in","speed":3})", + json_response)); + CHECK_TRUE(json_response.find(R"("success":true)") != std::string::npos); + CHECK_TRUE(g_ptz_call_count.load() == 2); + CHECK_TRUE(g_last_ptz_command.load() == ZOOM_IN); + CHECK_TRUE(g_last_ptz_speed.load() == 3); + + // A changed SDK file header invalidates the descriptor generation, but is + // not exposed as decoder config until its vendor-specific format is known. + emitEsPacket(kFileHeader, g_real_handle, {0x01, 0x02, 0x03}); + emitPFrame(); + CHECK_TRUE(!camera.waitEncodedFrame( + frame, cursor, std::chrono::milliseconds(10))); + emitIFrame(); + CHECK_TRUE(camera.waitEncodedFrame( + frame, cursor, std::chrono::milliseconds(50))); + CHECK_TRUE(frame.sequence == 1); + CHECK_TRUE(frame.codec_config_generation == 2); + CHECK_TRUE(frame.codec_config.empty()); + + constexpr int kConcurrentCallbacks = 8; + std::vector producers; + producers.reserve(kConcurrentCallbacks); + for (int i = 0; i < kConcurrentCallbacks; ++i) { + producers.emplace_back(emitPFrame, g_real_handle); + } + for (auto& producer : producers) { + producer.join(); + } + + size_t latest_cursor = 0; + cmvr::device::StreamFrameData latest; + CHECK_TRUE(camera.getLatestEncodedFrame(latest, latest_cursor)); + CHECK_TRUE(latest.stream_epoch == 1); + CHECK_TRUE(latest.sequence == static_cast(kConcurrentCallbacks + 1)); + CHECK_TRUE(latest_cursor == static_cast(kConcurrentCallbacks + 2)); + + for (int sequence = 2; sequence <= kConcurrentCallbacks + 1; ++sequence) { + CHECK_TRUE(camera.waitEncodedFrame( + frame, cursor, std::chrono::milliseconds(50))); + CHECK_TRUE(frame.stream_epoch == 1); + CHECK_TRUE(frame.sequence == static_cast(sequence)); + } + + camera.stopStreaming(); + emitIFrame(); + CHECK_TRUE(!camera.waitEncodedFrame( + frame, cursor, std::chrono::milliseconds(10))); + + CHECK_TRUE(camera.startStreaming()); + CHECK_TRUE(g_key_frame_request_count.load() == 3); + emitPFrame(); + emitIFrame(g_real_handle - 1); + CHECK_TRUE(!camera.waitEncodedFrame( + frame, cursor, std::chrono::milliseconds(10))); + emitIFrame(); + CHECK_TRUE(camera.waitEncodedFrame( + frame, cursor, std::chrono::milliseconds(50))); + CHECK_TRUE(frame.stream_epoch == 2); + CHECK_TRUE(frame.sequence == 0); + CHECK_TRUE(frame.codec_config_generation == 3); + + // The fake StopRealPlay invokes the SDK callback synchronously. stop() + // owns ctrl_mtx_ here, proving the callback neither takes that mutex nor + // publishes after the preview handle has been invalidated. + CHECK_TRUE(camera.stop()); + CHECK_TRUE(g_stop_callback_count.load() == 1); + CHECK_TRUE(!camera.waitEncodedFrame( + frame, cursor, std::chrono::milliseconds(10))); + return true; +} + +} // namespace + +extern "C" { + +BOOL NET_DVR_Init() +{ + return TRUE; +} + +BOOL NET_DVR_Cleanup() +{ + return TRUE; +} + +BOOL NET_DVR_SetConnectTime(DWORD, DWORD) +{ + return TRUE; +} + +BOOL NET_DVR_SetReconnect(DWORD, BOOL) +{ + return TRUE; +} + +BOOL NET_DVR_SetLogToFile(DWORD, char*, BOOL) +{ + return TRUE; +} + +LONG NET_DVR_Login_V40( + LPNET_DVR_USER_LOGIN_INFO, + LPNET_DVR_DEVICEINFO_V40) +{ + return 7; +} + +BOOL NET_DVR_Logout(LONG) +{ + return TRUE; +} + +DWORD NET_DVR_GetLastError() +{ + return 0; +} + +LONG NET_DVR_RealPlay_V40( + LONG, + LPNET_DVR_PREVIEWINFO, + REALDATACALLBACK, + void*) +{ + return g_real_handle; +} + +BOOL NET_DVR_SetESRealPlayCallBack( + LONG, + EsDataCallback callback, + void* user) +{ + std::lock_guard lock(g_fake_sdk_mutex); + g_es_data_callback = callback; + g_es_data_user = user; + return TRUE; +} + +BOOL NET_DVR_StopRealPlay(const LONG real_handle) +{ + EsDataCallback callback = nullptr; + void* user = nullptr; + { + std::lock_guard lock(g_fake_sdk_mutex); + callback = g_es_data_callback; + user = g_es_data_user; + } + + if (callback) { + std::vector idr{ + 0x00, 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0x21 + }; + NET_DVR_PACKET_INFO_EX packet{}; + packet.wWidth = 640; + packet.wHeight = 360; + packet.dwPacketType = kVideoIFrame; + packet.dwPacketSize = static_cast(idr.size()); + packet.pPacketBuffer = idr.data(); + callback(real_handle, &packet, user); + ++g_stop_callback_count; + } + + { + std::lock_guard lock(g_fake_sdk_mutex); + g_es_data_callback = nullptr; + g_es_data_user = nullptr; + } + return TRUE; +} + +BOOL NET_DVR_SaveRealData(LONG, char*) +{ + return TRUE; +} + +BOOL NET_DVR_StopSaveRealData(LONG) +{ + return TRUE; +} + +BOOL NET_DVR_MakeKeyFrame(LONG, LONG) +{ + ++g_key_frame_request_count; + return TRUE; +} + +BOOL NET_DVR_MakeKeyFrameSub(LONG, LONG) +{ + ++g_key_frame_request_count; + return TRUE; +} + +BOOL NET_DVR_PTZControlWithSpeed_Other( + LONG, + LONG, + const DWORD command, + const DWORD stop, + const DWORD speed) +{ + ++g_ptz_call_count; + g_last_ptz_command = command; + g_last_ptz_stop = stop; + g_last_ptz_speed = speed; + return TRUE; +} + +} // extern "C" + +int main() +{ + if (!testCallbackPublicationLifecycle()) { + return 1; + } + std::cout << "hikvision_camera_callback_test: PASS\n"; + return 0; +} diff --git a/cmvr-es/devices/camera/mechmind/include/mechmind_camera.h b/cmvr-es/devices/camera/mechmind/include/mechmind_camera.h index 6a8c225b..7a8628b6 100644 --- a/cmvr-es/devices/camera/mechmind/include/mechmind_camera.h +++ b/cmvr-es/devices/camera/mechmind/include/mechmind_camera.h @@ -30,6 +30,7 @@ namespace cmvr::device bool init() override; bool start() override; bool stop() override; + void getState(CameraState& state) override; void getRGBImage(cv::Mat& color, Rs2Intrinsics& intrinsics) override; void getDepthImage(cv::Mat& depth, Rs2Intrinsics& intrinsics) override; void getRGBDImages(cv::Mat &color, cv::Mat &depth, Rs2Intrinsics& intrinsics) override; diff --git a/cmvr-es/devices/camera/mechmind/src/mechmind_camera.cpp b/cmvr-es/devices/camera/mechmind/src/mechmind_camera.cpp index 39e30539..27f53738 100644 --- a/cmvr-es/devices/camera/mechmind/src/mechmind_camera.cpp +++ b/cmvr-es/devices/camera/mechmind/src/mechmind_camera.cpp @@ -50,6 +50,11 @@ bool MechmindCamera::init() { return true; } +void MechmindCamera::getState(CameraState& state) { + std::lock_guard lock(dev_mtx_); + state = state_; +} + bool MechmindCamera::start() { try { std::lock_guard lock(dev_mtx_); @@ -68,6 +73,7 @@ bool MechmindCamera::start() { return true; } catch (const std::exception& e) { + std::lock_guard lock(dev_mtx_); const string error_msg = "[MechmindCamera] (start): " + string(e.what()); CMVR_LOG(ERROR) << error_msg; state_.is_error = true; @@ -85,6 +91,7 @@ bool MechmindCamera::stop() { return true; } catch (const exception &e) { + std::lock_guard lock(dev_mtx_); const string error_msg = "[MechmindCamera] (stop): " + string(e.what()); CMVR_LOG(ERROR) << error_msg; state_.is_error = true; @@ -141,6 +148,7 @@ void MechmindCamera::getRGBImage(cv::Mat& color, Rs2Intrinsics& intrinsics) { } } catch (const exception &e) { + std::lock_guard lock(dev_mtx_); const string error_msg = "[MechmindCamera] (getRGBImage): " + string(e.what()); CMVR_LOG(ERROR) << error_msg; state_.is_error = true; @@ -175,6 +183,7 @@ void MechmindCamera::getDepthImage(cv::Mat& depth, Rs2Intrinsics& intrinsics) { depth = cv::Mat(depthMap.height(), depthMap.width(), CV_32FC1, depthMap.data()); } catch (const exception &e) { + std::lock_guard lock(dev_mtx_); const string error_msg = "[MechmindCamera] (getDepthImage): " + string(e.what()); CMVR_LOG(ERROR) << error_msg; state_.is_error = true; @@ -258,6 +267,7 @@ void MechmindCamera::getRGBDImages(cv::Mat& color, cv::Mat& depth, Rs2Intrinsics textured_plc_to_rgbd_(textured_pcl, color, depth); } catch (const exception &e) { + std::lock_guard lock(dev_mtx_); const string error_msg = "[MechmindCamera] (getRGBDImages): " + string(e.what()); CMVR_LOG(ERROR) << error_msg; state_.is_error = true; @@ -269,12 +279,14 @@ void MechmindCamera::getRGBDImages(cv::Mat& color, cv::Mat& depth, Rs2Intrinsics } void MechmindCamera::startRecording(const std::string& video_path) { + std::lock_guard lock(dev_mtx_); state_.is_error = true; state_.error_message = "startRecording is not implemented"; CMVR_LOG(ERROR) << "[MechmindCamera] (startRecording): " << state_.error_message; } void MechmindCamera::stopRecording() { + std::lock_guard lock(dev_mtx_); state_.is_error = true; state_.error_message = "stopRecording is not implemented"; CMVR_LOG(ERROR) << "[MechmindCamera] (stopRecording): " << state_.error_message; diff --git a/cmvr-es/devices/camera/mujoco_camera/include/mujoco_camera.h b/cmvr-es/devices/camera/mujoco_camera/include/mujoco_camera.h index 220e9936..6c3a3a0d 100644 --- a/cmvr-es/devices/camera/mujoco_camera/include/mujoco_camera.h +++ b/cmvr-es/devices/camera/mujoco_camera/include/mujoco_camera.h @@ -25,6 +25,7 @@ public: ~MujocoCamera() override = default; std::string typeName() const override { return "MujocoCamera"; } + void getState(CameraState& state) override; void setFovyDeg(double fovy_deg); void setConsumeNewFrameOnly(bool enable); diff --git a/cmvr-es/devices/camera/mujoco_camera/src/mujoco_camera.cpp b/cmvr-es/devices/camera/mujoco_camera/src/mujoco_camera.cpp index 05b44f91..82e63d77 100644 --- a/cmvr-es/devices/camera/mujoco_camera/src/mujoco_camera.cpp +++ b/cmvr-es/devices/camera/mujoco_camera/src/mujoco_camera.cpp @@ -11,6 +11,11 @@ namespace cmvr::device { MujocoCamera::MujocoCamera(FetchRgbdFn fetch_rgbd_fn) : fetch_rgbd_fn_(std::move(fetch_rgbd_fn)) {} +void MujocoCamera::getState(CameraState& state) { + std::lock_guard lock(mtx_); + state = state_; +} + void MujocoCamera::setFovyDeg(double fovy_deg) { std::lock_guard lock(mtx_); fovy_deg_ = fovy_deg; diff --git a/cmvr-es/devices/camera/realsense_camera/include/realsense_camera.h b/cmvr-es/devices/camera/realsense_camera/include/realsense_camera.h index 50a8eb7b..95d8002c 100644 --- a/cmvr-es/devices/camera/realsense_camera/include/realsense_camera.h +++ b/cmvr-es/devices/camera/realsense_camera/include/realsense_camera.h @@ -6,6 +6,7 @@ #define REALSENSE_CAMERA_H #include "../../uvc_camera/include/uvc_camera.h" +#include #include #include @@ -22,6 +23,7 @@ namespace cmvr::device{ bool init() override; bool start() override; bool stop() override; + void getState(CameraState& state) override; void getRGBImage(cv::Mat& color, Rs2Intrinsics& intrinsics) override; void getDepthImage(cv::Mat& depth, Rs2Intrinsics& intrinsics) override; void getRGBDImages(cv::Mat &color, cv::Mat &depth, Rs2Intrinsics& intrinsics) override; @@ -36,10 +38,12 @@ namespace cmvr::device{ int width, int height, int fps); static bool encodeFrameWithEncoder(std::shared_ptr& encoder,const cv::Mat& frame,std::vector& encoded_frame,bool& is_key); void getEncodedFrame(StreamFrameData& frame_data, size_t& index) override; + bool waitEncodedFrame(StreamFrameData& frame_data, size_t& index, std::chrono::milliseconds timeout) override; bool getLatestEncodedFrame(StreamFrameData& frame_data, size_t& next_index) override; bool startStreaming() override; void stopStreaming() override; + bool requestKeyFrame() override; Eigen::Vector3f get3DPointFromPixel(int u, int v) override; @@ -47,6 +51,9 @@ namespace cmvr::device{ rs2::frameset get_frameset(bool align); void streaming_worker_(); void recording_worker_(); + void join_stream_thread_() noexcept; + void join_recording_thread_() noexcept; + void cleanup_recording_resources_(bool finalize_file) noexcept; private: int fps_; int width_; @@ -103,9 +110,16 @@ namespace cmvr::device{ size_t recordingIndex_ = 0; size_t getImageIndex_ = 0; - bool is_streaming_running = false; - bool is_recording_running = false; + // Worker threads never read CameraState directly. Public lifecycle + // methods update CameraState under ctrl_mtx_, while these atomics form + // the cancellation/running handshake with the workers. + std::atomic capture_requested_{false}; + std::atomic capture_running_{false}; + std::atomic recording_requested_{false}; + std::atomic recording_running_{false}; int stream_count_ = 0; + std::atomic stream_epoch_{0}; + std::atomic codec_config_generation_{0}; cv::Mat latest_depth_; std::mutex depth_mtx_; diff --git a/cmvr-es/devices/camera/realsense_camera/src/realsense_camera.cpp b/cmvr-es/devices/camera/realsense_camera/src/realsense_camera.cpp index f5afa71d..b994588c 100644 --- a/cmvr-es/devices/camera/realsense_camera/src/realsense_camera.cpp +++ b/cmvr-es/devices/camera/realsense_camera/src/realsense_camera.cpp @@ -1,4 +1,6 @@ #include "common/base/logging/logger.h" +#include +#include // // Created by linbo on 2025/6/18. // @@ -140,10 +142,87 @@ RealsenseCamera::~RealsenseCamera() { } } +void RealsenseCamera::join_stream_thread_() noexcept { + auto thread = std::move(stream_thread_); + if (!thread || !thread->joinable()) { + return; + } + try { + if (thread->get_id() == std::this_thread::get_id()) { + CMVR_LOG(ERROR) << "[RealsenseCamera] refusing to join capture thread from itself"; + thread->detach(); + return; + } + thread->join(); + } catch (const std::exception& e) { + CMVR_LOG(ERROR) << "[RealsenseCamera] failed to join capture thread: " << e.what(); + if (thread->joinable()) { + thread->detach(); + } + } +} + +void RealsenseCamera::join_recording_thread_() noexcept { + auto thread = std::move(recording_thread_); + if (!thread || !thread->joinable()) { + return; + } + try { + if (thread->get_id() == std::this_thread::get_id()) { + CMVR_LOG(ERROR) << "[RealsenseCamera] refusing to join recording thread from itself"; + thread->detach(); + return; + } + thread->join(); + } catch (const std::exception& e) { + CMVR_LOG(ERROR) << "[RealsenseCamera] failed to join recording thread: " << e.what(); + if (thread->joinable()) { + thread->detach(); + } + } +} + +void RealsenseCamera::cleanup_recording_resources_(const bool finalize_file) noexcept { + if (packet_) { + av_packet_free(&packet_); + } + if (stream_ && stream_->codecpar->extradata) { + av_free(stream_->codecpar->extradata); + stream_->codecpar->extradata = nullptr; + stream_->codecpar->extradata_size = 0; + } + if (format_context_) { + if (!(format_context_->oformat->flags & AVFMT_NOFILE) && format_context_->pb) { + avio_closep(&format_context_->pb); + } + avformat_free_context(format_context_); + format_context_ = nullptr; + } + stream_ = nullptr; + + if (finalize_file && !current_video_path_.empty()) { + const std::string temp_path = current_video_path_ + ".temp"; + if (rename(temp_path.c_str(), current_video_path_.c_str()) != 0) { + state_.is_error = true; + state_.error_message = + "failed to rename temp file: " + temp_path + " -> " + current_video_path_; + CMVR_LOG(ERROR) << "[RealsenseCamera] (cleanup recording): " + << state_.error_message; + } + } + current_video_path_.clear(); +} + bool RealsenseCamera::init() { try { std::lock_guard lock(ctrl_mtx_); clear_error_(); + if (state_.is_opened || capture_requested_.load() || stream_thread_) { + state_.is_error = true; + state_.error_message = "camera must be stopped before reinitialization"; + CMVR_LOG(ERROR) << "[RealsenseCamera] (init): " << state_.error_message; + return false; + } state_.is_initialized = false; stream_frame_buffer_ = std::make_shared>(buffer_size_); @@ -184,6 +263,7 @@ bool RealsenseCamera::init() { return true; } catch (const exception &error){ + std::lock_guard lock(ctrl_mtx_); state_.is_initialized = false; CMVR_LOG(ERROR) << "realsense camera connect error:" << error.what(); state_.is_error = true; @@ -193,6 +273,11 @@ bool RealsenseCamera::init() { } +void RealsenseCamera::getState(CameraState& state) { + std::lock_guard lock(ctrl_mtx_); + state = state_; +} + bool RealsenseCamera::start() { std::lock_guard lock(ctrl_mtx_); clear_error_(); @@ -297,25 +382,29 @@ bool RealsenseCamera::start() { } bool RealsenseCamera::stop() { - //先停止录制再关闭摄像头 - if (state_.is_recording) { - try { - stopRecording(); - } catch (const std::exception& e) { - CMVR_LOG(WARNING) << "[RealsenseCamera] (stop): stopRecording failed: " << e.what(); - } - } std::lock_guard lock(ctrl_mtx_); clear_error_(); - if (!state_.is_opened || !state_.is_initialized) { - state_.is_opened = false; - return true; - } + + // Publish cancellation before stopping the pipeline. pipeline::stop() + // wakes a worker blocked in wait_for_frames(); the joins below guarantee + // that no thread can access this object after stop()/destruction returns. + const bool finalize_recording = state_.is_recording || recording_requested_.load(); + state_.is_recording = false; + state_.is_streaming = false; + stream_count_ = 0; + recording_requested_.store(false); + capture_requested_.store(false); + try { pipe_.stop(); } catch (const std::exception& e) { CMVR_LOG(WARNING) << "[RealsenseCamera] (stop): " << e.what(); } + + join_recording_thread_(); + join_stream_thread_(); + cleanup_recording_resources_(finalize_recording); + align_.reset(); pipe_ = rs2::pipeline(); state_.is_opened = false; @@ -323,13 +412,6 @@ bool RealsenseCamera::stop() { } void RealsenseCamera::getRGBImage(cv::Mat& color, Rs2Intrinsics& intrinsics) { - intrinsics.cx = intrinsics_.ppx; - intrinsics.cy = intrinsics_.ppy; - intrinsics.fx = intrinsics_.fx; - intrinsics.fy = intrinsics_.fy; - for (int i = 0; i < 5 ; i++) { - intrinsics.coeffs[i] = intrinsics_.coeffs[i]; - } try { std::lock_guard lock(ctrl_mtx_); clear_error_(); @@ -340,6 +422,13 @@ void RealsenseCamera::getRGBImage(cv::Mat& color, Rs2Intrinsics& intrinsics) { color.release(); return; } + intrinsics.cx = intrinsics_.ppx; + intrinsics.cy = intrinsics_.ppy; + intrinsics.fx = intrinsics_.fx; + intrinsics.fy = intrinsics_.fy; + for (int i = 0; i < 5 ; i++) { + intrinsics.coeffs[i] = intrinsics_.coeffs[i]; + } // 先尝试从队列取数据，没有的话再直接从摄像头读取 auto frame = stream_frame_buffer_->pop(getImageIndex_); if (frame.has_value()) { @@ -370,6 +459,7 @@ void RealsenseCamera::getRGBImage(cv::Mat& color, Rs2Intrinsics& intrinsics) { // CMVR_LOG(INFO) << "realsense get rgb frame successfully"; } catch (const std::exception& e) { + std::lock_guard lock(ctrl_mtx_); state_.is_error = true; state_.error_message = "[RealsenseCamera] (getRGBImage): Failed to get image: " + std::string(e.what()); CMVR_LOG(ERROR) << state_.error_message; @@ -378,6 +468,16 @@ void RealsenseCamera::getRGBImage(cv::Mat& color, Rs2Intrinsics& intrinsics) { } void RealsenseCamera::getDepthImage(cv::Mat& depth, Rs2Intrinsics& intrinsics) { + std::lock_guard lock(ctrl_mtx_); + clear_error_(); + if (!state_.is_opened) { + state_.is_error = true; + state_.error_message = "camera not opened"; + CMVR_LOG(ERROR) << "[RealsenseCamera] (getDepthImage): " << state_.error_message; + depth.release(); + return; + } + intrinsics.cx = intrinsics_.ppx; intrinsics.cy = intrinsics_.ppy; intrinsics.fx = intrinsics_.fx; @@ -424,6 +524,17 @@ void RealsenseCamera::getDepthImage(cv::Mat& depth, Rs2Intrinsics& intrinsics) { } void RealsenseCamera::getRGBDImages(cv::Mat &color, cv::Mat &depth, Rs2Intrinsics& intrinsics) { + std::lock_guard lock(ctrl_mtx_); + clear_error_(); + if (!state_.is_opened) { + state_.is_error = true; + state_.error_message = "camera not opened"; + CMVR_LOG(ERROR) << "[RealsenseCamera] (getRGBDImages): " << state_.error_message; + color.release(); + depth.release(); + return; + } + intrinsics.cx = intrinsics_.ppx; intrinsics.cy = intrinsics_.ppy; intrinsics.fx = intrinsics_.fx; @@ -498,15 +609,9 @@ void RealsenseCamera::startRecording(const std::string &video_path) { } auto cleanup_recording_resources = [this]() { - if (packet_) av_packet_free(&packet_); - if (stream_ && stream_->codecpar->extradata) av_free(stream_->codecpar->extradata); - if (format_context_) { - if (!(format_context_->oformat->flags & AVFMT_NOFILE) && format_context_->pb) avio_closep(&format_context_->pb); - avformat_free_context(format_context_); - } - stream_ = nullptr; - format_context_ = nullptr; + recording_requested_.store(false); state_.is_recording = false; + cleanup_recording_resources_(false); }; try { @@ -595,36 +700,40 @@ void RealsenseCamera::startRecording(const std::string &video_path) { cleanup_recording_resources(); return; } - //不在录像也不在流传输，但是采集线程没有退出时。 - if (!state_.is_streaming && !state_.is_recording) { - if (stream_thread_) { - if (stream_thread_->joinable()) { - stream_thread_->join(); - is_streaming_running = false; - } - stream_thread_.reset(); - } + // Reap a worker left behind by a previous capture session before + // starting a new one. + if (!capture_requested_.load() && stream_thread_) { + join_stream_thread_(); } + // 开启录像 state_.is_recording = true; + recording_requested_.store(true); + //开启流采集线程 if (!stream_thread_) { + capture_requested_.store(true); stream_thread_ = make_shared(&RealsenseCamera::streaming_worker_, this); //延时100ms，等待流线程获取图像 std::this_thread::sleep_for(std::chrono::milliseconds(100)); } + // 启动录像线程 frame_count_ = 0; if (recording_thread_) { - if (recording_thread_->joinable()) { - recording_thread_->join(); - is_recording_running = false; - } - recording_thread_.reset(); + recording_requested_.store(false); + join_recording_thread_(); + recording_requested_.store(true); } recording_thread_ = make_shared(&RealsenseCamera::recording_worker_, this); } catch (const std::exception& e) { + recording_requested_.store(false); + join_recording_thread_(); + if (stream_count_ == 0) { + capture_requested_.store(false); + join_stream_thread_(); + } cleanup_recording_resources(); state_.is_error = true; state_.error_message = "[RealsenseCamera] (startRecording): " + std::string(e.what()); @@ -646,41 +755,19 @@ void RealsenseCamera::stopRecording() { return; } - // 1. 停止录像线程 + // 1. Stop and join the recording worker before releasing its FFmpeg + // resources. The atomic is the worker's sole lifecycle signal. state_.is_recording = false; - if (recording_thread_ && recording_thread_->joinable()) { - recording_thread_->join(); - recording_thread_.reset(); - } + recording_requested_.store(false); + join_recording_thread_(); + cleanup_recording_resources_(true); - // 2. 清理FFmpeg资源 - if (packet_) { - av_packet_free(&packet_); - packet_ = nullptr; + // Recording and live streaming share one capture worker. Once the last + // user is gone, cancel and reap it as part of the lease release. + if (stream_count_ == 0) { + capture_requested_.store(false); + join_stream_thread_(); } - if (stream_ && stream_->codecpar->extradata) { - av_free(stream_->codecpar->extradata); - stream_->codecpar->extradata = nullptr; - stream_->codecpar->extradata_size = 0; - } - if (format_context_) { - if (!(format_context_->oformat->flags & AVFMT_NOFILE) && format_context_->pb) { - avio_closep(&format_context_->pb); // 关闭文件 - } - avformat_free_context(format_context_); // 释放格式上下文 - format_context_ = nullptr; - } - stream_ = nullptr; - - // 3. 重命名临时文件为目标文件 - std::string temp_path = current_video_path_ + ".temp"; - if (rename(temp_path.c_str(), current_video_path_.c_str()) != 0) { - state_.is_error = true; - state_.error_message = "failed to rename temp file: " + temp_path + " -> " + current_video_path_; - CMVR_LOG(ERROR) << "[RealsenseCamera] (stopRecording): " << state_.error_message; - return; - } - current_video_path_.clear(); } void RealsenseCamera::pauseRecording() { @@ -704,32 +791,36 @@ rs2::frameset RealsenseCamera::get_frameset(bool align) { } void RealsenseCamera::streaming_worker_() { + std::string failure; + capture_running_.store(true); + const uint64_t stream_epoch = stream_epoch_.fetch_add(1) + 1; + const uint32_t codec_config_generation = codec_config_generation_.fetch_add(1) + 1; + uint64_t stream_sequence = 0; + try { // 计算理论上每帧之间的间隔时间(毫秒) - const int frame_interval = 1000 / fps_; + const int frame_interval = 1000 / std::max(1, fps_); bool success = false; - is_streaming_running = true; - // 处于流传输或者录像状态时就不退出线程 - while (state_.is_streaming || state_.is_recording) { + // Streaming and recording share this producer. Lifecycle state is + // communicated exclusively through the atomic cancellation token. + while (capture_requested_.load()) { // 记录当前帧处理开始时间 auto frame_start_time = std::chrono::high_resolution_clock::now(); + const auto capture_monotonic = std::chrono::steady_clock::now(); + const auto capture_utc = std::chrono::system_clock::now(); rs2::frameset frames; frames = get_frameset(true); rs2::frame color_frame = frames.get_color_frame(); rs2::frame depth_frame = frames.get_depth_frame(); if (!color_frame) { - state_.is_error = true; - state_.error_message = "missing color frame"; - CMVR_LOG(ERROR) << "[RealsenseCamera]streaming_worker_: " << state_.error_message; + failure = "missing color frame"; break; } if (stream_mode_ == RGBD_MODE && !depth_frame) { - state_.is_error = true; - state_.error_message = "missing depth frame in RGBD mode"; - CMVR_LOG(ERROR) << "[RealsenseCamera]streaming_worker_: " << state_.error_message; + failure = "missing depth frame in RGBD mode"; break; } @@ -775,6 +866,24 @@ void RealsenseCamera::streaming_worker_() { frame_data.width = encode_width_; frame_data.height = encode_height_; frame_data.codec = codec_; + frame_data.stream_epoch = stream_epoch; + frame_data.sequence = stream_sequence++; + frame_data.capture_monotonic_ns = std::chrono::duration_cast( + capture_monotonic.time_since_epoch()).count(); + frame_data.capture_utc_ns = std::chrono::duration_cast( + capture_utc.time_since_epoch()).count(); + frame_data.pts = rgbEncoder_ ? rgbEncoder_->frame->pts : static_cast(frame_data.sequence); + frame_data.dts = frame_data.pts; + frame_data.time_base_num = 1; + frame_data.time_base_den = std::max(1, fps_); + frame_data.duration = 1; + frame_data.codec_config_generation = codec_config_generation; + if (frame_data.bKey && rgbEncoder_ && rgbEncoder_->codec_context && + rgbEncoder_->codec_context->extradata && rgbEncoder_->codec_context->extradata_size > 0) { + frame_data.codec_config.assign( + rgbEncoder_->codec_context->extradata, + rgbEncoder_->codec_context->extradata + rgbEncoder_->codec_context->extradata_size); + } stream_frame_buffer_->push(frame_data); } // 计算从帧开始到现在的总耗时 @@ -794,31 +903,42 @@ void RealsenseCamera::streaming_worker_() { } } - is_streaming_running = false; - // 线程结束时清空队列 - stream_frame_buffer_->clear(); - recordingIndex_ = 0; - getImageIndex_ = 0; - streamIndex_ = 0; } catch (const std::exception &e) { - // 线程结束时清空队列 + // pipe_.stop() intentionally interrupts wait_for_frames() during + // shutdown. Only report failures that happened while capture was + // still requested. + if (capture_requested_.load()) { + failure = e.what(); + } + } + + const bool unexpected_stop = capture_requested_.exchange(false); + capture_running_.store(false); + if (stream_frame_buffer_) { stream_frame_buffer_->clear(); - // 确保线程状态正确更新 - is_streaming_running = false; - state_.is_error = true; - state_.error_message = e.what(); - CMVR_LOG(ERROR) << "[RealsenseCamera]streaming_worker_ error:" << state_.error_message; + } + + if (unexpected_stop && !failure.empty()) { + CMVR_LOG(ERROR) << "[RealsenseCamera] streaming worker stopped: " << failure; } } void RealsenseCamera::recording_worker_() { - is_recording_running = true; - const int frame_interval = 1000 / fps_; + recording_running_.store(true); + const int frame_interval = 1000 / std::max(1, fps_); bool is_first_key = false; try { //保证当前采集线程正常运行 - while (state_.is_recording && is_streaming_running) { + while (recording_requested_.load()) { + if (!capture_running_.load()) { + if (!capture_requested_.load()) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + continue; + } + // 等待缓冲区有数据 if (stream_frame_buffer_->empty()) { std::this_thread::sleep_for(std::chrono::milliseconds(frame_interval)); @@ -886,8 +1006,8 @@ void RealsenseCamera::recording_worker_() { CMVR_LOG(ERROR) << "Recording thread error: " << e.what(); } - is_recording_running = false; - state_.is_recording = false; + recording_requested_.store(false); + recording_running_.store(false); } // 初始化单个编码器的通用函数 @@ -1071,6 +1191,9 @@ bool RealsenseCamera::encodeFrameWithEncoder(std::shared_ptr& return false; } + encoded_frame.clear(); + is_key = false; + // 确保输入帧尺寸匹配（不变） if (frame.cols != encoder->width || frame.rows != encoder->height) { CMVR_LOG(ERROR) << "Frame size does not match encoder dimensions"; @@ -1084,6 +1207,9 @@ bool RealsenseCamera::encodeFrameWithEncoder(std::shared_ptr& // 【修复3：设置递增的PTS，确保编码器正确处理I帧请求】 encoder->frame->pts = encoder->frame_pts++; // 分配唯一PTS + encoder->frame->pict_type = encoder->force_key_frame.exchange(false) + ? AV_PICTURE_TYPE_I + : AV_PICTURE_TYPE_NONE; // 根据cv::Mat的类型设置源格式（不变） AVPixelFormat src_pix_fmt; @@ -1147,9 +1273,6 @@ bool RealsenseCamera::encodeFrameWithEncoder(std::shared_ptr& if (encoder->packet->flags & AV_PKT_FLAG_KEY) { is_key = true; //CMVR_LOG(INFO) << "Encoded I frame (size: " << encoder->packet->size << " bytes)"; - } else { - is_key = false; - //CMVR_LOG(INFO) << "Encoded P frame (size: " << encoder->packet->size << " bytes)"; } // 预留足够空间，避免多次内存分配 @@ -1189,63 +1312,101 @@ void RealsenseCamera::getEncodedFrame(StreamFrameData& frame_data, size_t& index } } +bool RealsenseCamera::waitEncodedFrame( + StreamFrameData& frame_data, + size_t& index, + const std::chrono::milliseconds timeout) { + if (!stream_frame_buffer_) { + return false; + } + auto frame = stream_frame_buffer_->waitPop(index, timeout); + if (!frame) { + return false; + } + frame_data = std::move(*frame); + return !frame_data.rgbFrame.empty() || !frame_data.depthFrame.empty(); +} + bool RealsenseCamera::getLatestEncodedFrame(StreamFrameData& frame_data, size_t& next_index) { if (!stream_frame_buffer_) { return false; } - const size_t head = stream_frame_buffer_->getHead(); - if (head == 0) { - return false; - } - - size_t latest_index = head - 1; - auto frame = stream_frame_buffer_->pop(latest_index); + auto frame = stream_frame_buffer_->getLatest(next_index); if (!frame.has_value()) { return false; } frame_data = frame.value(); - next_index = latest_index; return true; } bool RealsenseCamera::startStreaming() { std::lock_guard lock(ctrl_mtx_); - //不在录像也不在流传输，但是采集线程没有退出时。 - if (!state_.is_streaming && !state_.is_recording) { - if (stream_thread_) { - if (stream_thread_->joinable()) { - stream_thread_->join(); - is_streaming_running = false; - } - stream_thread_.reset(); - } + clear_error_(); + if (!state_.is_initialized || !state_.is_opened) { + state_.is_error = true; + state_.error_message = "camera not opened"; + CMVR_LOG(ERROR) << "[RealsenseCamera] (startStreaming): " << state_.error_message; + return false; } + + // A completed worker remains joinable until it is reaped. Reap it before + // creating a replacement; otherwise destruction would call terminate(). + if (!capture_requested_.load() && stream_thread_) { + join_stream_thread_(); + } + //开启流采集线程 if (!stream_thread_) { + capture_requested_.store(true); state_.is_streaming = true; - stream_thread_ = make_shared(&RealsenseCamera::streaming_worker_, this); - - //延时100ms，等待流线程获取图像 - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + try { + stream_thread_ = make_shared(&RealsenseCamera::streaming_worker_, this); + } catch (const std::exception& e) { + capture_requested_.store(false); + state_.is_streaming = false; + state_.is_error = true; + state_.error_message = "failed to create capture thread: " + std::string(e.what()); + CMVR_LOG(ERROR) << "[RealsenseCamera] (startStreaming): " + << state_.error_message; + return false; + } } - stream_count_++; + ++stream_count_; + state_.is_streaming = true; return true; } void RealsenseCamera::stopStreaming() { std::lock_guard lock(ctrl_mtx_); - stream_count_--; + if (stream_count_ > 0) { + --stream_count_; + } if (stream_count_ == 0) { - // 当前已经没有正在使用的流了，编码采集线程状态修改 + // Recording may still own the shared producer. Otherwise release the + // final lease synchronously so no joinable thread is left behind. state_.is_streaming = false; + if (!recording_requested_.load()) { + capture_requested_.store(false); + join_stream_thread_(); + } } } +bool RealsenseCamera::requestKeyFrame() +{ + std::lock_guard lock(ctrl_mtx_); + if (!rgbEncoder_) { + return false; + } + rgbEncoder_->force_key_frame.store(true); + return true; +} + Eigen::Vector3f RealsenseCamera::get3DPointFromPixel(int u, int v) { diff --git a/cmvr-es/devices/camera/uvc_camera/CMakeLists.txt b/cmvr-es/devices/camera/uvc_camera/CMakeLists.txt index 77596fdf..0b45d15f 100644 --- a/cmvr-es/devices/camera/uvc_camera/CMakeLists.txt +++ b/cmvr-es/devices/camera/uvc_camera/CMakeLists.txt @@ -2,7 +2,19 @@ add_library(uvc_camera SHARED src/uvc_camera.cpp) target_include_directories(uvc_camera PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(uvc_camera PUBLIC glog opencv_core opencv_imgproc cmvr_es::proto) +target_link_libraries(uvc_camera + PUBLIC + glog + opencv_core + opencv_imgproc + cmvr_es::proto + PRIVATE + opencv_videoio + avcodec + avformat + avutil + swscale +) add_library(cmvr_es::device::uvc_camera ALIAS uvc_camera) install(TARGETS uvc_camera LIBRARY DESTINATION lib) diff --git a/cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h b/cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h index 9c035efa..6e5c0d05 100644 --- a/cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h +++ b/cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h @@ -5,6 +5,11 @@ #ifndef CMVR_ES_UVC_CAMERA_H #define CMVR_ES_UVC_CAMERA_H +#include +#include +#include +#include + #include "common/base/ring_buffer.h" #include "camera/abstract_camera.h" @@ -33,6 +38,7 @@ namespace cmvr::device { int height = 0; // 图像高度 int fps = 0; // 帧率 int64_t frame_pts = 0; + std::atomic force_key_frame{false}; bool bRunning = false; // 是否进行编码 AVCodecContext* codec_context = nullptr; // 编码器上下文 AVFrame* frame = nullptr; // 输入帧 @@ -73,6 +79,7 @@ namespace cmvr::device { ~UVCCamera() override; std::string typeName() const override { return "UVCCamera"; } + void getState(CameraState& state) override; bool init() override; bool start() override; bool stop() override; @@ -89,11 +96,23 @@ namespace cmvr::device { int width, int height, int fps); static bool encodeFrameWithEncoder(std::shared_ptr& encoder,const cv::Mat& frame,std::vector& encoded_frame,bool& is_key); void getEncodedFrame(StreamFrameData& frame_data, size_t& index) override; + bool waitEncodedFrame(StreamFrameData& frame_data, size_t& index, std::chrono::milliseconds timeout) override; bool getLatestEncodedFrame(StreamFrameData& frame_data, size_t& next_index) override; bool startStreaming() override; void stopStreaming() override; + bool requestKeyFrame() override; private: + // Public lifecycle operations are serialized by lifecycle_mtx_. Worker + // threads never acquire it, so they can always be collected without a + // lifecycle/control lock inversion. + bool open_capture_locked_(); + bool ensure_capture_worker_(); + bool stop_capture_worker_if_unused_(bool force); + void stop_recording_impl_(bool warn_if_inactive); + void cleanup_recording_resources_locked_(); + void reset_stream_metadata_(); + void set_worker_error_(const std::string& message); void streaming_worker_(); void recording_worker_(); @@ -106,13 +125,15 @@ namespace cmvr::device { cv::VideoCapture cap_; size_t buffer_size_; std::string codec_; - CameraMode mode_; + CameraMode mode_{PHOTO_MODE}; // std::shared_ptr current_image_; std::shared_ptr> stream_frame_buffer_; std::string current_video_path_; + std::mutex lifecycle_mtx_{}; std::mutex ctrl_mtx_{}; + std::mutex stream_metadata_mtx_{}; std::unique_ptr video_writer_; std::shared_ptr stream_thread_; std::shared_ptr recording_thread_; @@ -122,7 +143,6 @@ namespace cmvr::device { std::mutex frame_mutex_; // 保护最新帧的访问 std::string output_path_; - bool is_recording_ = false; // ffmpeg录像 AVFormatContext* format_context_ = nullptr; AVCodecContext* codec_context_ = nullptr; @@ -134,13 +154,15 @@ namespace cmvr::device { std::shared_ptr rgbEncoder_;//rgb图像编码 std::shared_ptr depthEncoder_;//深度图编码 - size_t streamIndex_ = 0; size_t recordingIndex_ = 0; - size_t getImageIndex_ = 0; - - bool is_streaming_running = false; - bool is_recording_running = false; + std::atomic capture_requested_{false}; + std::atomic capture_running_{false}; + std::atomic recording_requested_{false}; + std::atomic recording_running_{false}; int stream_count_ = 0; + uint64_t stream_epoch_ = 0; + uint64_t stream_sequence_ = 0; + uint32_t codec_config_generation_ = 0; config::UVCCameraConfig camera_; diff --git a/cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp b/cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp index dd6e80f3..bb540c51 100644 --- a/cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp +++ b/cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp @@ -1,4 +1,7 @@ #include "common/base/logging/logger.h" +#include +#include +#include // // Created by xtkuang on 2025/5/30. // @@ -48,15 +51,188 @@ UVCCamera::UVCCamera(const config::UVCCameraConfig& camera):camera_(camera) } UVCCamera::~UVCCamera() { stop(); - if (stream_thread_ && stream_thread_->joinable()) - stream_thread_->join(); - if (recording_thread_ && recording_thread_->joinable()) - recording_thread_->join(); +} + +void UVCCamera::getState(CameraState& state) { + std::lock_guard lock(ctrl_mtx_); + state = state_; +} + +bool UVCCamera::open_capture_locked_() { + if (cap_.isOpened()) { + return true; + } + + cap_.open(serial_, cv::CAP_V4L2); + this_thread::sleep_for(chrono::milliseconds(100)); + if (!cap_.isOpened()) { + state_.is_error = true; + state_.error_message = "Failed to open USB camera at index " + serial_; + CMVR_LOG(ERROR) << "[UVCCamera] (open_capture_locked_): " + << state_.error_message; + return false; + } + + cap_.set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G')); + + if (!cap_.set(cv::CAP_PROP_FRAME_WIDTH, width_)) { + state_.is_error = true; + state_.error_message = "set width failed"; + } else if (!cap_.set(cv::CAP_PROP_FRAME_HEIGHT, height_)) { + state_.is_error = true; + state_.error_message = "set height failed"; + } else if (!cap_.set(cv::CAP_PROP_FPS, fps_)) { + state_.is_error = true; + state_.error_message = "set fps failed"; + } else { + state_.width = width_; + state_.height = height_; + state_.fps = fps_; + return true; + } + + CMVR_LOG(ERROR) << "[UVCCamera] (open_capture_locked_): " + << state_.error_message; + cap_.release(); + return false; +} + +void UVCCamera::reset_stream_metadata_() { + { + std::lock_guard metadata_lock(stream_metadata_mtx_); + ++stream_epoch_; + stream_sequence_ = 0; + ++codec_config_generation_; + } + if (rgbEncoder_) { + rgbEncoder_->force_key_frame.store(true); + } +} + +void UVCCamera::set_worker_error_(const std::string& message) { + std::lock_guard lock(ctrl_mtx_); + state_.is_error = true; + state_.error_message = message; +} + +bool UVCCamera::ensure_capture_worker_() { + std::shared_ptr stale_worker; + { + std::lock_guard lock(ctrl_mtx_); + if (stream_thread_ && capture_requested_.load()) { + return true; + } + stale_worker = std::move(stream_thread_); + } + + if (stale_worker && stale_worker->joinable()) { + stale_worker->join(); + } + + std::lock_guard lock(ctrl_mtx_); + // A failed/finished worker may leave an open but unusable V4L2 backend. + // It is safe to reset only after that worker has been joined. + if (stale_worker && cap_.isOpened()) { + cap_.release(); + } + if (!state_.is_initialized || !state_.is_opened) { + state_.is_error = true; + state_.error_message = "camera not opened"; + return false; + } + if (!open_capture_locked_()) { + return false; + } + + // A producer restart while streaming is a discontinuity even if the + // logical subscriber lease did not change. + if (stream_count_ > 0) { + reset_stream_metadata_(); + } + capture_requested_.store(true); + try { + stream_thread_ = make_shared(&UVCCamera::streaming_worker_, this); + } catch (const std::exception& e) { + capture_requested_.store(false); + state_.is_error = true; + state_.error_message = + "failed to create capture thread: " + std::string(e.what()); + CMVR_LOG(ERROR) << "[UVCCamera] (ensure_capture_worker_): " + << state_.error_message; + return false; + } + return true; +} + +bool UVCCamera::stop_capture_worker_if_unused_(const bool force) { + std::shared_ptr worker; + std::shared_ptr> buffer; + { + std::lock_guard lock(ctrl_mtx_); + if (!force && (stream_count_ > 0 || recording_requested_.load())) { + return true; + } + + capture_requested_.store(false); + worker = std::move(stream_thread_); + buffer = stream_frame_buffer_; + } + + try { + if (worker && worker->joinable()) { + worker->join(); + } + } catch (const std::exception& e) { + set_worker_error_("failed to join capture thread: " + std::string(e.what())); + return false; + } + + capture_running_.store(false); + { + // The worker no longer touches either cap_ or its shared backend, so + // release is ordered after all waitAny()/retrieve() calls. + std::lock_guard lock(ctrl_mtx_); + if (cap_.isOpened()) { + cap_.release(); + } + } + if (buffer) { + buffer->clear(); + } + return true; +} + +void UVCCamera::cleanup_recording_resources_locked_() { + if (packet_) { + av_packet_free(&packet_); + } + if (stream_ && stream_->codecpar->extradata) { + av_free(stream_->codecpar->extradata); + stream_->codecpar->extradata = nullptr; + stream_->codecpar->extradata_size = 0; + } + if (format_context_) { + if (!(format_context_->oformat->flags & AVFMT_NOFILE) && + format_context_->pb) { + avio_closep(&format_context_->pb); + } + avformat_free_context(format_context_); + } + stream_ = nullptr; + format_context_ = nullptr; } bool UVCCamera::init() { + std::lock_guard lifecycle_lock(lifecycle_mtx_); std::lock_guard lock(ctrl_mtx_); clear_error_(); + if (state_.is_opened || stream_thread_ || recording_thread_ || + stream_count_ > 0 || recording_requested_.load()) { + state_.is_error = true; + state_.error_message = "cannot initialize while capture is active"; + CMVR_LOG(ERROR) << "[UVCCamera] (init): " << state_.error_message; + return false; + } state_.is_initialized = false; stream_frame_buffer_ = std::make_shared>(buffer_size_); @@ -65,37 +241,7 @@ bool UVCCamera::init() { cap_.release(); } - cap_.open(serial_, cv::CAP_V4L2); - this_thread::sleep_for(chrono::milliseconds(100)); - - if (!cap_.isOpened()) { - state_.is_error = true; - state_.error_message = "Failed to open USB camera at index " + serial_; - CMVR_LOG(ERROR) << "[UVCCamera] (init)" << state_.error_message; - return false; - } - - // 设置格式为MJPG - cap_.set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G')); - - if (!cap_.set(cv::CAP_PROP_FRAME_WIDTH, width_)) { - state_.is_error = true; - state_.error_message = "set width failed"; - CMVR_LOG(ERROR) << "[UVCCamera] (init): set width failed"; - return false; - } - state_.width = width_; - if (!cap_.set(cv::CAP_PROP_FRAME_HEIGHT, height_)) { - state_.is_error = true; - state_.error_message = "set height failed"; - CMVR_LOG(ERROR) << "[UVCCamera] (init): set height failed"; - return false; - } - state_.height = height_; - if (!cap_.set(cv::CAP_PROP_FPS, fps_)) { - state_.is_error = true; - state_.error_message = "set fps failed"; - CMVR_LOG(ERROR) << "[UVCCamera] (init): set fps failed"; + if (!open_capture_locked_()) { return false; } //初始化编码器 @@ -114,6 +260,7 @@ bool UVCCamera::init() { } bool UVCCamera::start() { + std::lock_guard lifecycle_lock(lifecycle_mtx_); std::lock_guard lock(ctrl_mtx_); clear_error_(); if (!state_.is_initialized) { @@ -123,40 +270,10 @@ bool UVCCamera::start() { return false; } if (state_.is_opened) { - CMVR_LOG(WARNING) << "[RealsenseCamera] (start): camera already started"; + CMVR_LOG(WARNING) << "[UVCCamera] (start): camera already started"; return true; } - cap_.open(serial_, cv::CAP_V4L2); - this_thread::sleep_for(chrono::milliseconds(100)); - - if (!cap_.isOpened()) { - state_.is_error = true; - state_.error_message = "Failed to open USB camera at index " + serial_; - CMVR_LOG(ERROR) << "[UVCCamera] (init)" << state_.error_message; - return false; - } - - // 设置格式为MJPG - cap_.set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G')); - - if (!cap_.set(cv::CAP_PROP_FRAME_WIDTH, width_)) { - state_.is_error = true; - state_.error_message = "set width failed"; - CMVR_LOG(ERROR) << "[UVCCamera] (init): set width failed"; - return false; - } - state_.width = width_; - if (!cap_.set(cv::CAP_PROP_FRAME_HEIGHT, height_)) { - state_.is_error = true; - state_.error_message = "set height failed"; - CMVR_LOG(ERROR) << "[UVCCamera] (init): set height failed"; - return false; - } - state_.height = height_; - if (!cap_.set(cv::CAP_PROP_FPS, fps_)) { - state_.is_error = true; - state_.error_message = "set fps failed"; - CMVR_LOG(ERROR) << "[UVCCamera] (init): set fps failed"; + if (!open_capture_locked_()) { return false; } state_.is_opened = true; @@ -164,42 +281,24 @@ bool UVCCamera::start() { } bool UVCCamera::stop() { - //先停止录制再关闭摄像头 - if (state_.is_recording) { - stopRecording(); - } - std::lock_guard lock(ctrl_mtx_); - clear_error_(); - try { - if (!state_.is_opened || !state_.is_initialized) { - CMVR_LOG(WARNING) << "[UVCCamera] (stop): Camera already closed"; - return true; - } - if (mode_ == VIDEO_MODE){ - state_.is_streaming = false; - if (stream_thread_->joinable()) { - stream_thread_->join(); - stream_thread_.reset(); - stream_thread_ = nullptr; - } - } + std::lock_guard lifecycle_lock(lifecycle_mtx_); - if (cap_.isOpened()) { - cap_.release(); - } + if (mode_ == VIDEO_MODE) { + stop_recording_impl_(false); + } + { + std::lock_guard lock(ctrl_mtx_); + clear_error_(); + stream_count_ = 0; + state_.is_streaming = false; state_.is_opened = false; - return true; - } - catch (exception &e) { - CMVR_LOG(ERROR) << "[UVCCamera] (stop): " << e.what(); - state_.is_error = true; - state_.error_message = e.what(); - return false; } + return stop_capture_worker_if_unused_(true); } void UVCCamera::getRGBImage(cv::Mat& color, Rs2Intrinsics& intrinsics) { + (void)intrinsics; std::lock_guard lock(ctrl_mtx_); clear_error_(); if (mode_ == PHOTO_MODE) { @@ -231,6 +330,8 @@ void UVCCamera::getRGBImage(cv::Mat& color, Rs2Intrinsics& intrinsics) } void UVCCamera::getDepthImage(cv::Mat& depth, Rs2Intrinsics& intrinsics) { + (void)intrinsics; + std::lock_guard lock(ctrl_mtx_); state_.is_error = true; state_.error_message = "getDepthImage unsupported usage"; CMVR_LOG(ERROR) << "[UVCCamera] (getDepthImage): " << state_.error_message; @@ -238,6 +339,8 @@ void UVCCamera::getDepthImage(cv::Mat& depth, Rs2Intrinsics& intrinsics) { } void UVCCamera::getRGBDImages(cv::Mat &color, cv::Mat &depth, Rs2Intrinsics& intrinsics) { + (void)intrinsics; + std::lock_guard lock(ctrl_mtx_); state_.is_error = true; state_.error_message = "getRGBDImages unsupported usage"; CMVR_LOG(ERROR) << "[UVCCamera] (getRGBDImages): " << state_.error_message; @@ -246,7 +349,8 @@ void UVCCamera::getRGBDImages(cv::Mat &color, cv::Mat &depth, Rs2Intrinsics& int } void UVCCamera::startRecording(const std::string &video_path) { - std::lock_guard lock(ctrl_mtx_); + std::lock_guard lifecycle_lock(lifecycle_mtx_); + std::unique_lock lock(ctrl_mtx_); clear_error_(); if (mode_ != VIDEO_MODE) { state_.is_error = true; @@ -268,14 +372,8 @@ void UVCCamera::startRecording(const std::string &video_path) { } auto cleanup_recording_resources = [this]() { - if (packet_) av_packet_free(&packet_); - if (stream_ && stream_->codecpar->extradata) av_free(stream_->codecpar->extradata); - if (format_context_) { - if (!(format_context_->oformat->flags & AVFMT_NOFILE) && format_context_->pb) avio_closep(&format_context_->pb); - avformat_free_context(format_context_); - } - stream_ = nullptr; - format_context_ = nullptr; + cleanup_recording_resources_locked_(); + recording_requested_.store(false); state_.is_recording = false; }; @@ -365,36 +463,38 @@ void UVCCamera::startRecording(const std::string &video_path) { cleanup_recording_resources(); return; } - //不在录像也不在流传输，但是采集线程没有退出时。 - if (!state_.is_streaming && !state_.is_recording) { - if (stream_thread_) { - if (stream_thread_->joinable()) { - stream_thread_->join(); - is_streaming_running = false; - } - stream_thread_.reset(); - } - } - // 开启录像 - state_.is_recording = true; - //开启流采集线程 - if (!stream_thread_) { - stream_thread_ = make_shared(&UVCCamera::streaming_worker_, this); - //延时100ms，等待流线程获取图像 - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - // 启动录像线程 - if (recording_thread_) { - if (recording_thread_->joinable()) { - recording_thread_->join(); - is_recording_running = false; - } - recording_thread_.reset(); - } + recordingIndex_ = stream_frame_buffer_ + ? stream_frame_buffer_->getHead() + : 0; frame_count_ = 0; - recording_thread_ = make_shared(&UVCCamera::recording_worker_, this); + recording_requested_.store(true); + state_.is_recording = true; + lock.unlock(); + if (!ensure_capture_worker_()) { + lock.lock(); + cleanup_recording_resources(); + return; + } + + lock.lock(); + try { + recording_thread_ = + make_shared(&UVCCamera::recording_worker_, this); + } catch (const std::exception& e) { + cleanup_recording_resources(); + state_.is_error = true; + state_.error_message = + "failed to create recording thread: " + std::string(e.what()); + CMVR_LOG(ERROR) << "[UVCCamera] (startRecording): " + << state_.error_message; + lock.unlock(); + stop_capture_worker_if_unused_(false); + } } catch (const std::exception& e) { + if (!lock.owns_lock()) { + lock.lock(); + } cleanup_recording_resources(); state_.is_error = true; state_.error_message = "[UVCCamera] (startRecording): " + std::string(e.what()); @@ -403,54 +503,70 @@ void UVCCamera::startRecording(const std::string &video_path) { } void UVCCamera::stopRecording() { - std::lock_guard lock(ctrl_mtx_); - clear_error_(); - if (mode_ != VIDEO_MODE) { - state_.is_error = true; - state_.error_message = "stopRecording only supports VIDEO_MODE"; - CMVR_LOG(ERROR) << "[UVCCamera] (stopRecording): " << state_.error_message; - return; - } - if (!state_.is_recording) { - CMVR_LOG(WARNING) << "[UVCCamera] (stopRecording): not recording"; - return; - } + std::lock_guard lifecycle_lock(lifecycle_mtx_); + stop_recording_impl_(true); +} - // 1. 停止录像线程 - state_.is_recording = false; - if (recording_thread_ && recording_thread_->joinable()) { - recording_thread_->join(); - recording_thread_.reset(); - } - - // 2. 清理FFmpeg资源 - if (packet_) { - av_packet_free(&packet_); - packet_ = nullptr; - } - if (stream_ && stream_->codecpar->extradata) { - av_free(stream_->codecpar->extradata); - stream_->codecpar->extradata = nullptr; - stream_->codecpar->extradata_size = 0; - } - if (format_context_) { - if (!(format_context_->oformat->flags & AVFMT_NOFILE) && format_context_->pb) { - avio_closep(&format_context_->pb); // 关闭文件 +void UVCCamera::stop_recording_impl_(const bool warn_if_inactive) { + std::shared_ptr worker; + std::string completed_video_path; + { + std::lock_guard lock(ctrl_mtx_); + clear_error_(); + if (mode_ != VIDEO_MODE) { + if (warn_if_inactive) { + state_.is_error = true; + state_.error_message = + "stopRecording only supports VIDEO_MODE"; + CMVR_LOG(ERROR) << "[UVCCamera] (stopRecording): " + << state_.error_message; + } + return; + } + if (!state_.is_recording && !recording_thread_) { + if (warn_if_inactive) { + CMVR_LOG(WARNING) + << "[UVCCamera] (stopRecording): not recording"; + } + return; } - avformat_free_context(format_context_); // 释放格式上下文 - format_context_ = nullptr; - } - stream_ = nullptr; - // 3. 重命名临时文件为目标文件 - std::string temp_path = current_video_path_ + ".temp"; - if (rename(temp_path.c_str(), current_video_path_.c_str()) != 0) { - state_.is_error = true; - state_.error_message = "failed to rename temp file: " + temp_path + " -> " + current_video_path_; - CMVR_LOG(ERROR) << "[UVCCamera] (stopRecording): " << state_.error_message; - return; + recording_requested_.store(false); + state_.is_recording = false; + worker = std::move(recording_thread_); + completed_video_path = current_video_path_; } - current_video_path_.clear(); + + try { + if (worker && worker->joinable()) { + worker->join(); + } + } catch (const std::exception& e) { + set_worker_error_( + "failed to join recording thread: " + std::string(e.what())); + } + recording_running_.store(false); + + { + std::lock_guard lock(ctrl_mtx_); + cleanup_recording_resources_locked_(); + if (!completed_video_path.empty()) { + const std::string temp_path = completed_video_path + ".temp"; + if (rename(temp_path.c_str(), completed_video_path.c_str()) != 0) { + state_.is_error = true; + state_.error_message = + "failed to rename temp file: " + temp_path + " -> " + + completed_video_path; + CMVR_LOG(ERROR) << "[UVCCamera] (stopRecording): " + << state_.error_message; + } + } + current_video_path_.clear(); + } + + // The capture producer is shared with streaming. Releasing the recording + // lease only stops it when no stream lease remains. + stop_capture_worker_if_unused_(false); } void UVCCamera::pauseRecording() { @@ -462,23 +578,50 @@ void UVCCamera::resumeRecording() { } void UVCCamera::streaming_worker_() { - + std::shared_ptr> buffer; + std::vector capture_streams; + { + std::lock_guard lock(ctrl_mtx_); + buffer = stream_frame_buffer_; + // VideoCapture is a reference-counted handle. stop() does not release + // cap_ until this worker exits, so this local handle has sole access to + // the backend's frame operations. + capture_streams.push_back(cap_); + } + capture_running_.store(true); try { // 计算理论上每帧之间的间隔时间(毫秒) - const int frame_interval = 1000 / fps_; + const int frame_interval = 1000 / std::max(1, fps_); bool success = false; - is_streaming_running = true; cv::Mat frame; - int64_t frame_count = 0; - // 处于流传输或者录像状态时就不退出线程 - while (state_.is_streaming || state_.is_recording) { + std::vector ready_streams; + constexpr int64_t kCapturePollTimeoutNs = 100'000'000; + while (capture_requested_.load()) { // 记录当前帧处理开始时间 auto frame_start_time = std::chrono::high_resolution_clock::now(); - if (!cap_.read(frame) || frame.empty()) { - state_.is_error = true; - state_.error_message = "failed to read frame"; - CMVR_LOG(ERROR) << "[UVCCamera]streaming_worker_: " << state_.error_message; + const auto capture_monotonic = std::chrono::steady_clock::now(); + const auto capture_utc = std::chrono::system_clock::now(); + ready_streams.clear(); + const bool frame_ready = cv::VideoCapture::waitAny( + capture_streams, ready_streams, kCapturePollTimeoutNs); + if (!capture_requested_.load()) { + break; + } + if (!frame_ready) { + continue; + } + if (ready_streams.empty() || ready_streams.front() != 0 || + !capture_streams.front().retrieve(frame) || frame.empty()) { + if (capture_requested_.load()) { + set_worker_error_("failed to retrieve frame"); + CMVR_LOG(ERROR) + << "[UVCCamera]streaming_worker_: " + << "failed to retrieve frame"; + } + break; + } + if (!capture_requested_.load()) { break; } @@ -505,7 +648,36 @@ void UVCCamera::streaming_worker_() { frame_data.width = encode_width_; frame_data.height = encode_height_; frame_data.codec = codec_; - stream_frame_buffer_->push(frame_data); + frame_data.capture_monotonic_ns = std::chrono::duration_cast( + capture_monotonic.time_since_epoch()).count(); + frame_data.capture_utc_ns = std::chrono::duration_cast( + capture_utc.time_since_epoch()).count(); + frame_data.time_base_num = 1; + frame_data.time_base_den = std::max(1, fps_); + frame_data.duration = 1; + if (frame_data.bKey && rgbEncoder_ && rgbEncoder_->codec_context && + rgbEncoder_->codec_context->extradata && rgbEncoder_->codec_context->extradata_size > 0) { + frame_data.codec_config.assign( + rgbEncoder_->codec_context->extradata, + rgbEncoder_->codec_context->extradata + rgbEncoder_->codec_context->extradata_size); + } + // Keep metadata allocation and publication in one critical + // section. Once reset_stream_metadata_() returns, no old + // epoch frame can be published afterwards. + { + std::lock_guard metadata_lock(stream_metadata_mtx_); + frame_data.stream_epoch = stream_epoch_; + frame_data.sequence = stream_sequence_++; + frame_data.codec_config_generation = + codec_config_generation_; + frame_data.pts = rgbEncoder_ + ? rgbEncoder_->frame->pts + : static_cast(frame_data.sequence); + frame_data.dts = frame_data.pts; + if (buffer) { + buffer->push(frame_data); + } + } } else { @@ -524,50 +696,48 @@ void UVCCamera::streaming_worker_() { std::this_thread::sleep_for(std::chrono::milliseconds(sleep_time)); } } - - is_streaming_running = false; - // 线程结束时清空队列 - stream_frame_buffer_->clear(); - recordingIndex_ = 0; - getImageIndex_ = 0; - streamIndex_ = 0; } catch (const std::exception &e) { - // 线程结束时清空队列 - stream_frame_buffer_->clear(); - // 确保线程状态正确更新 - is_streaming_running = false; - state_.is_error = true; - state_.error_message = e.what(); - CMVR_LOG(ERROR) << "[UVCCamera]streaming_worker_ error:" << state_.error_message; + if (capture_requested_.load()) { + set_worker_error_(e.what()); + CMVR_LOG(ERROR) << "[UVCCamera]streaming_worker_ error:" + << e.what(); + } + } + catch (...) { + if (capture_requested_.load()) { + set_worker_error_("unknown capture worker error"); + CMVR_LOG(ERROR) + << "[UVCCamera]streaming_worker_: unknown worker error"; + } + } + capture_requested_.store(false); + capture_running_.store(false); + if (buffer) { + buffer->clear(); } } void UVCCamera::recording_worker_() { - is_recording_running = true; - const int frame_interval = 1000 / fps_; + std::shared_ptr> buffer; + { + std::lock_guard lock(ctrl_mtx_); + buffer = stream_frame_buffer_; + } + recording_running_.store(true); bool is_first_key = false; try { - //保证当前采集线程正常运行 - while (state_.is_recording && is_streaming_running) { - // 等待缓冲区有数据 - if (stream_frame_buffer_->empty()) { - std::this_thread::sleep_for(std::chrono::milliseconds(frame_interval)); - continue; - } - - // 1. 从缓冲区取编码好的帧 - auto frame_data_opt = stream_frame_buffer_->pop(recordingIndex_); - + while (recording_requested_.load()) { + auto frame_data_opt = buffer + ? buffer->waitPop(recordingIndex_, std::chrono::milliseconds(100)) + : std::nullopt; if (!frame_data_opt.has_value()) { - std::this_thread::sleep_for(std::chrono::milliseconds(frame_interval)); continue; } - StreamFrameData frame_data = frame_data_opt.value(); + StreamFrameData frame_data = std::move(*frame_data_opt); // 2. 验证编码数据有效性 if (frame_data.rgbFrame.empty()) { - std::this_thread::sleep_for(std::chrono::milliseconds(frame_interval)); continue; } @@ -606,19 +776,22 @@ void UVCCamera::recording_worker_() { if (av_interleaved_write_frame(format_context_, packet_) < 0) { CMVR_LOG(ERROR) << "写入第" << frame_count_ << "帧失败"; } - - std::this_thread::sleep_for(std::chrono::milliseconds(33)); } // 7. 写入文件尾（完成封装） - av_write_trailer(format_context_); + if (format_context_) { + av_write_trailer(format_context_); + } } catch (const std::exception& e) { + set_worker_error_("recording worker error: " + std::string(e.what())); CMVR_LOG(ERROR) << "录像线程错误: " << e.what(); + } catch (...) { + set_worker_error_("unknown recording worker error"); + CMVR_LOG(ERROR) << "录像线程未知错误"; } - is_recording_running = false; - state_.is_recording = false; + recording_running_.store(false); } // 初始化单个编码器的通用函数 @@ -727,6 +900,9 @@ bool UVCCamera::encodeFrameWithEncoder(std::shared_ptr& encod return false; } + encoded_frame.clear(); + is_key = false; + // 确保输入帧尺寸匹配（不变） if (frame.cols != encoder->width || frame.rows != encoder->height) { CMVR_LOG(ERROR) << "Frame size does not match encoder dimensions"; @@ -735,6 +911,9 @@ bool UVCCamera::encodeFrameWithEncoder(std::shared_ptr& encod // 【修复3：设置递增的PTS，确保编码器正确处理I帧请求】 encoder->frame->pts = encoder->frame_pts++; // 分配唯一PTS + encoder->frame->pict_type = encoder->force_key_frame.exchange(false) + ? AV_PICTURE_TYPE_I + : AV_PICTURE_TYPE_NONE; // 根据cv::Mat的类型设置源格式（不变） AVPixelFormat src_pix_fmt; @@ -797,9 +976,6 @@ bool UVCCamera::encodeFrameWithEncoder(std::shared_ptr& encod if (encoder->packet->flags & AV_PKT_FLAG_KEY) { is_key = true; //CMVR_LOG(INFO) << "Encoded I frame (size: " << encoder->packet->size << " bytes)"; - } else { - is_key = false; - //CMVR_LOG(INFO) << "Encoded P frame (size: " << encoder->packet->size << " bytes)"; } // 预留足够空间，避免多次内存分配 @@ -815,11 +991,16 @@ bool UVCCamera::encodeFrameWithEncoder(std::shared_ptr& encod // 验证帧有效性（NALU起始码、元数据） if (!encoded_frame.empty()) { // 检查NALU起始码 - bool has_start_code = false; - if ((encoded_frame[0] == 0 && encoded_frame[1] == 0 && encoded_frame[2] == 1) || - (encoded_frame[0] == 0 && encoded_frame[1] == 0 && encoded_frame[2] == 0 && encoded_frame[3] == 1)) { - has_start_code = true; - } + const bool has_three_byte_start_code = + encoded_frame.size() >= 3 && + encoded_frame[0] == 0 && encoded_frame[1] == 0 && + encoded_frame[2] == 1; + const bool has_four_byte_start_code = + encoded_frame.size() >= 4 && + encoded_frame[0] == 0 && encoded_frame[1] == 0 && + encoded_frame[2] == 0 && encoded_frame[3] == 1; + const bool has_start_code = + has_three_byte_start_code || has_four_byte_start_code; if (!has_start_code) { CMVR_LOG(ERROR) << "Invalid frame: no NALU start code!"; return false; @@ -832,66 +1013,133 @@ bool UVCCamera::encodeFrameWithEncoder(std::shared_ptr& encod } void UVCCamera::getEncodedFrame(StreamFrameData& frame_data, size_t& index) { + std::shared_ptr> buffer; + { + std::lock_guard lock(ctrl_mtx_); + buffer = stream_frame_buffer_; + } + if (!buffer) { + return; + } // 环形队列取数据的index由接口传入 - auto frame = stream_frame_buffer_->pop(index); + auto frame = buffer->pop(index); if (frame.has_value()) { frame_data = frame.value(); } } +bool UVCCamera::waitEncodedFrame( + StreamFrameData& frame_data, + size_t& index, + const std::chrono::milliseconds timeout) { + std::shared_ptr> buffer; + { + std::lock_guard lock(ctrl_mtx_); + buffer = stream_frame_buffer_; + } + if (!buffer) { + return false; + } + auto frame = buffer->waitPop(index, timeout); + if (!frame) { + return false; + } + frame_data = std::move(*frame); + return !frame_data.rgbFrame.empty() || !frame_data.depthFrame.empty(); +} + bool UVCCamera::getLatestEncodedFrame(StreamFrameData& frame_data, size_t& next_index) { - if (!stream_frame_buffer_) { + std::shared_ptr> buffer; + { + std::lock_guard lock(ctrl_mtx_); + buffer = stream_frame_buffer_; + } + if (!buffer) { return false; } - const size_t head = stream_frame_buffer_->getHead(); - if (head == 0) { - return false; - } - - size_t latest_index = head - 1; - auto frame = stream_frame_buffer_->pop(latest_index); + auto frame = buffer->getLatest(next_index); if (!frame.has_value()) { return false; } frame_data = frame.value(); - next_index = latest_index; return true; } bool UVCCamera::startStreaming() { - std::lock_guard lock(ctrl_mtx_); - //不在录像也不在流传输，但是采集线程没有退出时。 - if (!state_.is_streaming && !state_.is_recording) { - if (stream_thread_) { - if (stream_thread_->joinable()) { - stream_thread_->join(); - is_streaming_running = false; - } - stream_thread_.reset(); + std::lock_guard lifecycle_lock(lifecycle_mtx_); + bool first_lease = false; + bool producer_already_running = false; + { + std::lock_guard lock(ctrl_mtx_); + clear_error_(); + if (mode_ != VIDEO_MODE) { + state_.is_error = true; + state_.error_message = + "startStreaming only supports VIDEO_MODE"; + CMVR_LOG(ERROR) << "[UVCCamera] (startStreaming): " + << state_.error_message; + return false; + } + if (!state_.is_initialized || !state_.is_opened) { + state_.is_error = true; + state_.error_message = "camera not opened"; + CMVR_LOG(ERROR) << "[UVCCamera] (startStreaming): " + << state_.error_message; + return false; } - } - //开启流采集线程 - if (!stream_thread_) { - state_.is_streaming = true; - stream_thread_ = make_shared(&UVCCamera::streaming_worker_, this); - //延时100ms，等待流线程获取图像 - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + first_lease = stream_count_ == 0; + producer_already_running = + stream_thread_ && capture_requested_.load(); + ++stream_count_; + state_.is_streaming = true; + } + + // If recording already owns the producer, starting a new externally + // visible stream still begins a fresh metadata epoch. A newly created + // producer resets it inside ensure_capture_worker_(). + if (first_lease && producer_already_running) { + reset_stream_metadata_(); + } + + if (!ensure_capture_worker_()) { + std::lock_guard lock(ctrl_mtx_); + if (stream_count_ > 0) { + --stream_count_; + } + if (stream_count_ == 0) { + state_.is_streaming = false; + } + return false; } - stream_count_++; return true; } void UVCCamera::stopStreaming() { - std::lock_guard lock(ctrl_mtx_); - stream_count_--; - if (stream_count_ == 0) + std::lock_guard lifecycle_lock(lifecycle_mtx_); { - // 当前已经没有正在使用的流了，编码采集线程状态修改 - state_.is_streaming = false; + std::lock_guard lock(ctrl_mtx_); + if (stream_count_ > 0) { + --stream_count_; + } + if (stream_count_ == 0) { + state_.is_streaming = false; + } } + // Recording is an independent lease on the same producer. + stop_capture_worker_if_unused_(false); +} + +bool UVCCamera::requestKeyFrame() +{ + std::lock_guard lock(ctrl_mtx_); + if (!rgbEncoder_) { + return false; + } + rgbEncoder_->force_key_frame.store(true); + return true; } diff --git a/cmvr-es/devices/microphone/abstract_microphone.h b/cmvr-es/devices/microphone/abstract_microphone.h index 6a4cdcb7..619acf99 100644 --- a/cmvr-es/devices/microphone/abstract_microphone.h +++ b/cmvr-es/devices/microphone/abstract_microphone.h @@ -6,6 +6,7 @@ #define CMVR_ES_ABSTRACT_MICROPHONE_H #pragma once +#include #include #include "devices/abstract_device.h" @@ -27,6 +28,14 @@ namespace cmvr::device{ virtual bool startStreaming() {return true;} virtual void stopStreaming() {} virtual void getEncodedFrame(AudioStreamFrameData& frame_data, size_t& index) {} + virtual bool waitEncodedFrame( + AudioStreamFrameData& frame_data, + size_t& index, + std::chrono::milliseconds timeout) { + (void)timeout; + getEncodedFrame(frame_data, index); + return !frame_data.data.empty(); + } virtual bool getLatestEncodedFrame(AudioStreamFrameData& frame_data, size_t& next_index) { return false; } diff --git a/cmvr-es/devices/microphone/ffmpeg_microphone/include/ffmpeg_microphone.h b/cmvr-es/devices/microphone/ffmpeg_microphone/include/ffmpeg_microphone.h index 0e17046d..cd41d881 100644 --- a/cmvr-es/devices/microphone/ffmpeg_microphone/include/ffmpeg_microphone.h +++ b/cmvr-es/devices/microphone/ffmpeg_microphone/include/ffmpeg_microphone.h @@ -43,6 +43,7 @@ namespace cmvr::device { bool startStreaming() override; void stopStreaming() override; void getEncodedFrame(AudioStreamFrameData& frame_data, size_t& index) override; + bool waitEncodedFrame(AudioStreamFrameData& frame_data, size_t& index, std::chrono::milliseconds timeout) override; bool getLatestEncodedFrame(AudioStreamFrameData& frame_data, size_t& next_index) override; private: @@ -51,6 +52,8 @@ namespace cmvr::device { void closeFFmpeg(); bool startCapture_(bool write_file); void stopCapture_(); + bool recoverStreamingCapture_(int read_error); + static int interruptCallback_(void* opaque); AudioStreamFormat currentStreamFormat_() const; void pushEncodedPacket_(const AVPacket* packet); @@ -70,6 +73,7 @@ namespace cmvr::device { std::string output_file; std::string format_name; std::atomic is_capturing_{false}; + std::atomic interrupt_requested_{false}; std::atomic write_output_file_{false}; std::atomic is_paused; std::mutex pause_mutex; @@ -82,10 +86,18 @@ namespace cmvr::device { std::string input_device_; std::shared_ptr audio_thread_; + // Serializes FFmpeg context start/stop and keeps a new shared lease from + // starting until the previous capture thread has exited and been joined. + std::mutex capture_mutex_; std::mutex mtx_; std::shared_ptr> stream_frame_buffer_; + // Protected by mtx_. The audio thread snapshots these values while it + // assigns metadata to an encoded frame. int stream_count_ = 0; size_t buffer_size_ = 256; + uint64_t stream_epoch_ = 0; + uint64_t stream_sequence_ = 0; + uint32_t codec_config_generation_ = 0; config::FFMpegMicroPhoneConfig config_; }; diff --git a/cmvr-es/devices/microphone/ffmpeg_microphone/src/ffmpeg_microphone.cpp b/cmvr-es/devices/microphone/ffmpeg_microphone/src/ffmpeg_microphone.cpp index da212f5b..8b58f987 100644 --- a/cmvr-es/devices/microphone/ffmpeg_microphone/src/ffmpeg_microphone.cpp +++ b/cmvr-es/devices/microphone/ffmpeg_microphone/src/ffmpeg_microphone.cpp @@ -108,6 +108,10 @@ bool ffmpegMicroPhone::start() bool ffmpegMicroPhone::stop() { + // Keep the lifecycle lock until FFmpeg has observed the interrupt request, + // the capture thread has exited, and all contexts have been released. + // A concurrent start cannot revive the device halfway through that cleanup. + std::lock_guard capture_lock(capture_mutex_); { lock_guard lock(mtx_); stream_count_ = 0; @@ -121,7 +125,10 @@ bool ffmpegMicroPhone::stop() void ffmpegMicroPhone::pause() { is_paused = true; - state_.is_error = false; + { + lock_guard lock(mtx_); + state_.is_error = false; + } CMVR_LOG(INFO) << "[ffmpegMicroPhone] (pause): Success, id=" << id_; } @@ -129,7 +136,10 @@ void ffmpegMicroPhone::resume() { is_paused = false; pause_cv.notify_one(); - state_.is_error = false; + { + lock_guard lock(mtx_); + state_.is_error = false; + } CMVR_LOG(INFO) << "[ffmpegMicroPhone] (resume): Success, id=" << id_; } @@ -141,33 +151,43 @@ void ffmpegMicroPhone::getState(MicrophoneState& state) void ffmpegMicroPhone::startRecording(const std::string& outputFilePath) { - lock_guard lock(mtx_); - if (state_.is_recording) { - CMVR_LOG(ERROR) << "[ffmpegMicroPhone] (startRecording): microphone is already recording"; - return; - } - if (stream_count_ > 0 || is_capturing_) { - state_.is_error = true; - state_.error_message = "cannot start recording while microphone streaming is active"; - CMVR_LOG(ERROR) << "[ffmpegMicroPhone] " << state_.error_message; - return; - } + std::lock_guard capture_lock(capture_mutex_); + { + lock_guard lock(mtx_); + if (state_.is_recording) { + CMVR_LOG(ERROR) << "[ffmpegMicroPhone] (startRecording): microphone is already recording"; + return; + } + if (stream_count_ > 0 || is_capturing_) { + state_.is_error = true; + state_.error_message = "cannot start recording while microphone streaming is active"; + CMVR_LOG(ERROR) << "[ffmpegMicroPhone] " << state_.error_message; + return; + } - size_t dot_pos = outputFilePath.find_last_of('.'); - format_name = dot_pos == std::string::npos ? "wav" : outputFilePath.substr(dot_pos + 1); - output_file = outputFilePath; + size_t dot_pos = outputFilePath.find_last_of('.'); + format_name = dot_pos == std::string::npos ? "wav" : outputFilePath.substr(dot_pos + 1); + output_file = outputFilePath; + } if (!startCapture_(true)) { + lock_guard lock(mtx_); state_.is_error = true; state_.error_message = "FFmpeg init error"; return; } - state_.is_recording = true; + { + lock_guard lock(mtx_); + state_.is_recording = true; + state_.is_error = false; + state_.error_message.clear(); + } } void ffmpegMicroPhone::stopRecording() { + std::lock_guard capture_lock(capture_mutex_); { lock_guard lock(mtx_); if (!state_.is_recording) { @@ -193,41 +213,64 @@ int ffmpegMicroPhone::getVolume() bool ffmpegMicroPhone::startStreaming() { - lock_guard lock(mtx_); - if (state_.is_recording) { - state_.is_error = true; - state_.error_message = "cannot start streaming while microphone recording is active"; - CMVR_LOG(ERROR) << "[ffmpegMicroPhone] " << state_.error_message; - return false; - } - if (stream_count_ == 0) { - stream_frame_buffer_->clear(); - format_name = "pcm_s16le"; - output_file.clear(); - if (!startCapture_(false)) { + std::lock_guard capture_lock(capture_mutex_); + { + lock_guard lock(mtx_); + if (state_.is_recording) { state_.is_error = true; - state_.error_message = "FFmpeg init error"; + state_.error_message = "cannot start streaming while microphone recording is active"; + CMVR_LOG(ERROR) << "[ffmpegMicroPhone] " << state_.error_message; return false; } + + if (stream_count_ > 0 && is_capturing_.load(std::memory_order_acquire)) { + ++stream_count_; + state_.is_running = true; + state_.is_error = false; + state_.error_message.clear(); + return true; + } + + stream_frame_buffer_->clear(); + ++stream_epoch_; + stream_sequence_ = 0; + ++codec_config_generation_; + format_name = "pcm_s16le"; + output_file.clear(); + } + + if (!startCapture_(false)) { + lock_guard lock(mtx_); + state_.is_error = true; + state_.error_message = "FFmpeg init error"; + return false; + } + + { + lock_guard lock(mtx_); + // capture_mutex_ prevents another start/stop transition from changing + // the lease count while the first capture is being initialized. + ++stream_count_; + state_.is_running = true; + state_.is_error = false; + state_.error_message.clear(); } - ++stream_count_; - state_.is_running = true; - state_.is_error = false; - state_.error_message.clear(); return true; } void ffmpegMicroPhone::stopStreaming() { - bool should_stop = false; + std::lock_guard capture_lock(capture_mutex_); + bool last_lease = false; { lock_guard lock(mtx_); - if (stream_count_ > 0) { - --stream_count_; + if (stream_count_ == 0) { + return; } - should_stop = stream_count_ == 0 && !state_.is_recording; + --stream_count_; + last_lease = stream_count_ == 0; } - if (should_stop) { + if (last_lease) { stopCapture_(); } } @@ -243,55 +286,95 @@ void ffmpegMicroPhone::getEncodedFrame(AudioStreamFrameData& frame_data, size_t& } } +bool ffmpegMicroPhone::waitEncodedFrame( + AudioStreamFrameData& frame_data, + size_t& index, + const std::chrono::milliseconds timeout) +{ + if (!stream_frame_buffer_) { + return false; + } + auto frame = stream_frame_buffer_->waitPop(index, timeout); + if (!frame) { + return false; + } + frame_data = std::move(*frame); + return !frame_data.data.empty(); +} + bool ffmpegMicroPhone::getLatestEncodedFrame(AudioStreamFrameData& frame_data, size_t& next_index) { if (!stream_frame_buffer_) { return false; } - const size_t head = stream_frame_buffer_->getHead(); - if (head == 0) { - return false; - } - - size_t latest_index = head - 1; - auto frame = stream_frame_buffer_->pop(latest_index); + auto frame = stream_frame_buffer_->getLatest(next_index); if (!frame.has_value()) { return false; } frame_data = frame.value(); - next_index = latest_index; return true; } bool ffmpegMicroPhone::startCapture_(bool write_file) { - if (is_capturing_) { + // The caller holds capture_mutex_ for the complete transition. + if (is_capturing_.load(std::memory_order_acquire)) { return true; } - write_output_file_ = write_file; - if (!initFFmpeg()) { + // Collect a worker that ended because of an unrecoverable input error + // before replacing its std::thread object or FFmpeg contexts. + if (audio_thread_) { + if (audio_thread_->joinable()) { + audio_thread_->join(); + } + audio_thread_.reset(); closeFFmpeg(); + } + interrupt_requested_.store(false, std::memory_order_release); + write_output_file_.store(write_file, std::memory_order_release); + if (!initFFmpeg()) { + interrupt_requested_.store(true, std::memory_order_release); + closeFFmpeg(); + write_output_file_.store(false, std::memory_order_release); return false; } - is_capturing_ = true; - is_paused = false; - state_.is_running = true; - audio_thread_ = std::make_shared(&ffmpegMicroPhone::audioThread, this); + is_capturing_.store(true, std::memory_order_release); + is_paused.store(false, std::memory_order_release); + try { + audio_thread_ = std::make_shared(&ffmpegMicroPhone::audioThread, this); + } catch (const std::exception& e) { + CMVR_LOG(ERROR) << "[ffmpegMicroPhone] Could not start capture thread: " << e.what(); + interrupt_requested_.store(true, std::memory_order_release); + is_capturing_.store(false, std::memory_order_release); + closeFFmpeg(); + write_output_file_.store(false, std::memory_order_release); + return false; + } + { + lock_guard lock(mtx_); + state_.is_running = true; + } return true; } void ffmpegMicroPhone::stopCapture_() { - if (!is_capturing_) { + // AVIOInterruptCB is FFmpeg's supported way to abort a blocking input + // operation. Set it before clearing the run flag so av_read_frame() can + // return AVERROR_EXIT and the thread can reach the join below. + interrupt_requested_.store(true, std::memory_order_release); + const bool was_capturing = is_capturing_.exchange(false, std::memory_order_acq_rel); + is_paused.store(false, std::memory_order_release); + pause_cv.notify_all(); + + if (!was_capturing && !audio_thread_) { + lock_guard lock(mtx_); + state_.is_running = false; return; } - is_capturing_ = false; - is_paused = false; - pause_cv.notify_all(); - if (audio_thread_) { if (audio_thread_->joinable()) { audio_thread_->join(); @@ -299,16 +382,70 @@ void ffmpegMicroPhone::stopCapture_() audio_thread_.reset(); } - if (write_output_file_ && output_fmt_ctx) { + if (write_output_file_.load(std::memory_order_acquire) && output_fmt_ctx) { av_write_trailer(output_fmt_ctx); } closeFFmpeg(); - write_output_file_ = false; + write_output_file_.store(false, std::memory_order_release); lock_guard lock(mtx_); state_.is_running = false; } +int ffmpegMicroPhone::interruptCallback_(void* opaque) +{ + const auto* microphone = static_cast(opaque); + return microphone && + microphone->interrupt_requested_.load(std::memory_order_acquire) + ? 1 + : 0; +} + +bool ffmpegMicroPhone::recoverStreamingCapture_(const int read_error) +{ + if (write_output_file_.load(std::memory_order_acquire)) { + return false; + } + + const std::string read_error_message = + "audio input read failed: " + avErrorToString(read_error); + { + lock_guard lock(mtx_); + state_.is_error = true; + state_.error_message = read_error_message + "; reconnecting"; + } + CMVR_LOG(ERROR) << "[ffmpegMicroPhone] " << read_error_message + << "; reconnecting input"; + + auto retry_delay = std::chrono::milliseconds(100); + constexpr auto kMaximumRetryDelay = std::chrono::milliseconds(2000); + while (is_capturing_.load(std::memory_order_acquire) && + !interrupt_requested_.load(std::memory_order_acquire)) { + closeFFmpeg(); + if (initFFmpeg()) { + { + lock_guard lock(mtx_); + ++stream_epoch_; + stream_sequence_ = 0; + ++codec_config_generation_; + state_.is_running = true; + state_.is_error = false; + state_.error_message.clear(); + } + CMVR_LOG(INFO) << "[ffmpegMicroPhone] Audio input reconnected"; + return true; + } + + std::unique_lock wait_lock(pause_mutex); + pause_cv.wait_for(wait_lock, retry_delay, [this] { + return interrupt_requested_.load(std::memory_order_acquire) || + !is_capturing_.load(std::memory_order_acquire); + }); + retry_delay = std::min(retry_delay * 2, kMaximumRetryDelay); + } + return false; +} + bool ffmpegMicroPhone::initFFmpeg() { int err = 0; @@ -334,6 +471,15 @@ bool ffmpegMicroPhone::initFFmpeg() CMVR_LOG(INFO) << "[ffmpegMicroPhone] Open input device: format=" << input_format_name << ", device=" << input_device_; + input_fmt_ctx = avformat_alloc_context(); + if (!input_fmt_ctx) { + CMVR_LOG(ERROR) << "[ffmpegMicroPhone] Could not allocate input format context"; + av_dict_free(&input_options); + return false; + } + input_fmt_ctx->interrupt_callback.callback = &ffmpegMicroPhone::interruptCallback_; + input_fmt_ctx->interrupt_callback.opaque = this; + err = avformat_open_input(&input_fmt_ctx, input_device_.c_str(), input_fmt, &input_options); av_dict_free(&input_options); if (err < 0) { @@ -488,22 +634,49 @@ void ffmpegMicroPhone::audioThread() CMVR_LOG(ERROR) << "[ffmpegMicroPhone] Could not allocate packets"; av_packet_free(&input_pkt); av_packet_free(&output_pkt); + is_capturing_.store(false, std::memory_order_release); + lock_guard lock(mtx_); + state_.is_running = false; + state_.is_error = true; + state_.error_message = "could not allocate capture packets"; return; } - while (is_capturing_) { - if (is_paused) { + while (is_capturing_.load(std::memory_order_acquire)) { + if (is_paused.load(std::memory_order_acquire)) { unique_lock lock(pause_mutex); - pause_cv.wait(lock, [this] { return !is_paused || !is_capturing_; }); + pause_cv.wait(lock, [this] { + return !is_paused.load(std::memory_order_acquire) || + !is_capturing_.load(std::memory_order_acquire); + }); continue; } const int read_ret = av_read_frame(input_fmt_ctx, input_pkt); if (read_ret < 0) { + if (interrupt_requested_.load(std::memory_order_acquire)) { + break; + } if (read_ret == AVERROR(EAGAIN)) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); continue; } + av_packet_unref(input_pkt); + if (recoverStreamingCapture_(read_ret)) { + continue; + } + if (interrupt_requested_.load(std::memory_order_acquire) || + !is_capturing_.load(std::memory_order_acquire)) { + break; + } + { + lock_guard lock(mtx_); + state_.is_error = true; + state_.error_message = + "audio input read failed: " + avErrorToString(read_ret); + } + CMVR_LOG(ERROR) << "[ffmpegMicroPhone] Audio capture stopped: " + << avErrorToString(read_ret); break; } @@ -574,22 +747,33 @@ void ffmpegMicroPhone::audioThread() } } - avcodec_send_frame(audio_codec_ctx, nullptr); - while (avcodec_receive_packet(audio_codec_ctx, output_pkt) == 0) { - if (write_output_file_ && output_fmt_ctx && audio_st) { - av_packet_rescale_ts(output_pkt, audio_codec_ctx->time_base, audio_st->time_base); - output_pkt->stream_index = audio_st->index; - if (av_interleaved_write_frame(output_fmt_ctx, output_pkt) < 0) { - av_packet_unref(output_pkt); - break; + if (audio_codec_ctx) { + avcodec_send_frame(audio_codec_ctx, nullptr); + while (avcodec_receive_packet(audio_codec_ctx, output_pkt) == 0) { + if (write_output_file_ && output_fmt_ctx && audio_st) { + av_packet_rescale_ts(output_pkt, audio_codec_ctx->time_base, audio_st->time_base); + output_pkt->stream_index = audio_st->index; + if (av_interleaved_write_frame(output_fmt_ctx, output_pkt) < 0) { + av_packet_unref(output_pkt); + break; + } } + pushEncodedPacket_(output_pkt); + av_packet_unref(output_pkt); } - pushEncodedPacket_(output_pkt); - av_packet_unref(output_pkt); } av_packet_free(&input_pkt); av_packet_free(&output_pkt); + if (!interrupt_requested_.load(std::memory_order_acquire)) { + is_capturing_.store(false, std::memory_order_release); + lock_guard lock(mtx_); + state_.is_running = false; + if (!state_.is_error) { + state_.is_error = true; + state_.error_message = "audio capture thread exited unexpectedly"; + } + } } AudioStreamFormat ffmpegMicroPhone::currentStreamFormat_() const @@ -605,6 +789,8 @@ AudioStreamFormat ffmpegMicroPhone::currentStreamFormat_() const case AV_CODEC_ID_PCM_S16LE: case AV_CODEC_ID_PCM_S16BE: return AudioStreamFormat::PCM; + case AV_CODEC_ID_OPUS: + return AudioStreamFormat::OPUS; default: return AudioStreamFormat::UNKNOWN; } @@ -612,18 +798,54 @@ AudioStreamFormat ffmpegMicroPhone::currentStreamFormat_() const void ffmpegMicroPhone::pushEncodedPacket_(const AVPacket* packet) { - if (!packet || !packet->data || packet->size <= 0 || stream_count_ <= 0 || !stream_frame_buffer_) { + if (!packet || !packet->data || packet->size <= 0 || !stream_frame_buffer_) { return; } + uint64_t stream_epoch = 0; + uint64_t stream_sequence = 0; + uint32_t codec_config_generation = 0; + { + lock_guard lock(mtx_); + if (stream_count_ <= 0) { + return; + } + stream_epoch = stream_epoch_; + stream_sequence = stream_sequence_++; + codec_config_generation = codec_config_generation_; + } + AudioStreamFrameData frame; + const auto capture_monotonic = std::chrono::steady_clock::now(); + const auto capture_utc = std::chrono::system_clock::now(); frame.data.assign(packet->data, packet->data + packet->size); frame.sample_rate = sample_rate_; frame.channels = channels_; frame.format = currentStreamFormat_(); frame.codec = audio_codec_ctx && audio_codec_ctx->codec ? audio_codec_ctx->codec->name : "pcm_s16le"; frame.pts = packet->pts; + frame.dts = packet->dts; frame.nb_samples = audio_frame ? audio_frame->nb_samples : 0; + frame.stream_epoch = stream_epoch; + frame.sequence = stream_sequence; + frame.capture_monotonic_ns = std::chrono::duration_cast( + capture_monotonic.time_since_epoch()).count(); + frame.capture_utc_ns = std::chrono::duration_cast( + capture_utc.time_since_epoch()).count(); + frame.time_base_num = audio_codec_ctx && audio_codec_ctx->time_base.num > 0 + ? audio_codec_ctx->time_base.num + : 1; + frame.time_base_den = audio_codec_ctx && audio_codec_ctx->time_base.den > 0 + ? audio_codec_ctx->time_base.den + : std::max(1, sample_rate_); + frame.duration = packet->duration > 0 ? packet->duration : frame.nb_samples; + frame.codec_config_generation = codec_config_generation; + if (frame.sequence == 0 && audio_codec_ctx && audio_codec_ctx->extradata && + audio_codec_ctx->extradata_size > 0) { + frame.codec_config.assign( + audio_codec_ctx->extradata, + audio_codec_ctx->extradata + audio_codec_ctx->extradata_size); + } stream_frame_buffer_->push(frame); } diff --git a/cmvr-es/devices/state_define.h b/cmvr-es/devices/state_define.h index 495f3cfa..8906347d 100644 --- a/cmvr-es/devices/state_define.h +++ b/cmvr-es/devices/state_define.h @@ -41,6 +41,7 @@ namespace cmvr::device{ MP3 = 1, AAC = 2, WAV = 3, + OPUS = 4, UNKNOWN = 99 }; @@ -51,7 +52,18 @@ namespace cmvr::device{ AudioStreamFormat format = AudioStreamFormat::PCM; std::string codec = "pcm_s16le"; int64_t pts = 0; + int64_t dts = 0; int nb_samples = 0; + uint64_t stream_epoch = 0; + uint64_t sequence = 0; + int64_t capture_monotonic_ns = 0; + int64_t capture_utc_ns = 0; + int32_t time_base_num = 1; + int32_t time_base_den = 44100; + int64_t duration = 0; + bool discontinuity = false; + uint32_t codec_config_generation = 0; + std::vector codec_config; }; // ------------------------------------- robot ------------------------------------- diff --git a/cmvr-es/hardware/README.md b/cmvr-es/hardware/README.md new file mode 100644 index 00000000..a24472db --- /dev/null +++ b/cmvr-es/hardware/README.md @@ -0,0 +1,89 @@ +# Hardware 模块开发指南 + +`hardware/` 提供串口等可复用的底层传输能力。设备命令、寄存器含义和厂商协议应保留在 `devices/` 的具体后端中。 + +返回[项目总览](../../README.md)。 + +## 当前组件 + +| 文件 | 作用 | +| --- | --- | +| [`abstract_serial_transport.h`](include/abstract_serial_transport.h) | 可替换、可测试的串口传输接口 | +| [`posix_serial_transport.h`](include/posix_serial_transport.h) | Linux POSIX 串口实现，支持超时和任意正波特率 | +| [`serial_interface.h`](include/serial_interface.h) | 现有 RS485/CAN 寄存器辅助接口 | +| [`esp32_serial_port.h`](include/esp32_serial_port.h) | ESP32/舵机串口辅助实现 | + +新设备优先依赖 `AbstractSerialTransport`，通过构造或 setter 注入实现。这样单元测试可以使用 fake transport，不需要真实 `/dev/tty*`。 + +## 底层与设备层边界 + +适合放在 `hardware/`： + +- 打开、关闭和查询传输状态； +- 有限超时的字节读写； +- 输入缓冲清理； +- OS 错误转换和统一错误文本； +- 与设备语义无关的帧读写工具。 + +应放在 `devices///`： + +- 帧头、命令码、寄存器地址和校验规则； +- 重试、握手和设备状态机； +- 设备 ID、关节、传感器和错误码含义； +- 后台接收线程及其与设备状态的同步。 + +## 新增传输实现 + +1. 实现 `AbstractSerialTransport` 或为新总线定义同样窄的抽象； +2. 析构函数必须可靠释放 fd、handle 和后台线程； +3. `open()` 失败后对象保持可再次打开； +4. `close()` 应幂等； +5. 所有阻塞读取必须支持有限 timeout； +6. `lastError()` 返回最近一次操作的可诊断错误； +7. 更新 [`CMakeLists.txt`](CMakeLists.txt) 和 `cmvr_es::hardware` target； +8. 使用 pseudo terminal、socketpair 或 fake 实现编写无设备测试。 + +## 并发与事务 + +`PosixSerialTransport` 使用互斥锁保护单次 `open/close/read/write`。这不等于一个“write command + read response”复合事务天然不会与其他线程交错。 + +具体驱动必须： + +- 由一个 I/O 线程独占 transport，或在驱动层为完整请求/响应加锁； +- 在关闭前先通知并 join 接收线程； +- 不持有设备状态锁执行长时间 I/O； +- 超时后清理残留输入，避免下一请求读到旧响应； +- 不让回调在 transport 锁内反向调用设备； +- 为部分写、短读、EINTR、EAGAIN 和设备拔出定义行为。 + +## 权限和部署 + +Linux 设备通常需要： + +- 将运行用户加入 `dialout`、`video` 或设备专用组； +- 使用 udev rule 固定权限和稳定设备别名； +- CAN 接口在服务启动前完成 `ip link` 配置； +- 不依赖开发机上的临时 `chmod 777`； +- 在日志中记录逻辑设备名，不输出密码和完整敏感报文。 + +## 新增总线类型 + +新增 CAN、TCP、USB 等通用传输时： + +1. 定义只描述字节/帧传输的抽象接口； +2. 将 Linux/SDK 实现与抽象分离； +3. 让设备后端依赖抽象而不是全局单例； +4. 提供 fake 实现和故障注入； +5. 记录线程安全级别和调用顺序； +6. 更新安装依赖及 systemd 权限说明。 + +## 测试清单 + +- [ ] 打开、重复打开、关闭和重复关闭 +- [ ] 非法路径和非法参数 +- [ ] 完整写、部分写和对端断开 +- [ ] 精确长度读取、短读和超时 +- [ ] EINTR/EAGAIN 重试 +- [ ] I/O 期间关闭 +- [ ] 多线程调用或显式拒绝并发 +- [ ] 析构后没有 fd 和线程泄漏 diff --git a/cmvr-es/main.cpp b/cmvr-es/main.cpp index 4affd835..6ec9c25d 100644 --- a/cmvr-es/main.cpp +++ b/cmvr-es/main.cpp @@ -16,6 +16,7 @@ #include "manager/device_manager/include/device_manager.h" #include "manager/task_manager/include/task_manager.h" #include "task/grpc_server_task/include/grpc_server_task.h" +#include "task/quic_edge_task/include/quic_edge_task.h" namespace { @@ -113,6 +114,7 @@ int main(int argc, char* argv[]) cmvr::device::DeviceManager::getInstance(device_manager_root.device_manager()); cmvr::task::registerGrpcServerTaskFactory(); + cmvr::task::registerQuicEdgeTaskFactory(); if (app_config.task_manager_config_file().empty()) { CMVR_LOG(ERROR) << "TaskManager config file is empty"; diff --git a/cmvr-es/manager/README.md b/cmvr-es/manager/README.md new file mode 100644 index 00000000..f261195d --- /dev/null +++ b/cmvr-es/manager/README.md @@ -0,0 +1,226 @@ +# Manager 模块开发指南 + +`manager/` 负责组织设备、任务和协议无关媒体源。Manager 管理对象生命周期和调度，不实现厂商协议，也不实现平台 wire protocol。 + +返回[项目总览](../../README.md)。 + +## 当前管理器 + +| 目录 | CMake target | 职责 | +| --- | --- | --- | +| [`device_manager/`](device_manager/) | `cmvr_es::device_manager` | 按配置创建、初始化、查询和批量启停设备 | +| [`task_manager/`](task_manager/) | `cmvr_es::task_manager` | 创建任务、校验运行模式、统一启停和调度周期任务 | +| [`media_source_hub/`](media_source_hub/) | `cmvr_es::media_source_hub`、`cmvr_es::device_media_source_adapter` | 实时媒体源注册、按需启停和多消费者分发 | + +`manager/` 当前没有聚合 `CMakeLists.txt`，三个子目录由 [`../CMakeLists.txt`](../CMakeLists.txt) 分别加入。新增 manager 时必须显式更新该文件。 + +## 进程生命周期 + +当前 [`../main.cpp`](../main.cpp) 的顺序是： + +1. 加载根配置并设置全局配置根； +2. 构造 `DeviceManager`； +3. 创建启用的设备并调用 `device->init()`； +4. 注册 gRPC、QUIC TaskFactory creator； +5. 构造 `TaskManager` 并调用启用任务的 `init()`； +6. `startRunTask()` 启动任务和周期调度线程； +7. 收到 SIGINT/SIGTERM 后调用 `TaskManager::stopRunTask()`。 + +重要限制： + +- DeviceManager 构造不会自动调用全部设备的 `start()`； +- 当前主退出路径没有调用 `DeviceManager::stop()`； +- `SystemService/StopAll` 会调用 DeviceManager stop； +- `DeviceManager::destroyInstance()` 不调用设备 stop，销毁前必须先显式停止； +- `TaskManager::destroyInstance()` 会调用 `stopRunTask()`，但 manager 未处于 running 状态时该调用会直接返回； +- DeviceManager 和 TaskManager 都是首次配置生效的单例，不支持热加载。 + +## DeviceManager + +关键文件： + +- [`device_manager/include/device_manager.h`](device_manager/include/device_manager.h) +- [`device_manager/include/device_factory.h`](device_manager/include/device_factory.h) +- [`device_manager/src/device_factory.cpp`](device_manager/src/device_factory.cpp) +- [`../devices/README.md`](../devices/README.md) + +### 增加现有类别的新后端 + +例如增加一种摄像头： + +1. 扩展类别配置 Proto 的 `oneof backend`； +2. 实现对应抽象设备； +3. 修改 `CameraFactory::create()`； +4. 增加 CMake target； +5. 在摄像头集合配置中增加实例； +6. 在 DeviceManager 配置中增加相同 ID 的条目。 + +这种扩展通常不修改全局 `DeviceFactory`，因为 `DEVICE_TYPE_CAMERA` 已经路由到 CameraFactory。 + +### 增加全新设备类别 + +还需要： + +1. 扩展 `DeviceConfigEntry::DeviceType`； +2. 扩展 `DeviceKind` 和字符串映射； +3. 在 `DeviceFactory::DeviceFactory()` 注册 creator； +4. 扩展 DeviceManager 的设备类型日志映射； +5. 为 `getDevice()` 增加显式模板实例化； +6. 在 device_manager target 链接新设备 target； +7. 如需平台访问，增加 API Proto、gRPC service 和系统设备类型映射。 + +`DeviceFactory::registerCreator()` 虽然是 public，但 factory 是 DeviceManager 的私有成员，当前不是运行时插件入口。新增全局设备类别仍需修改 `device_factory.cpp`。 + +### 容器和顺序约束 + +- 配置启用条目按配置顺序创建并 `init()`； +- 有初始化依赖的设备应把依赖项写在使用方之前； +- `start()`、`stop()` 遍历 `unordered_map`，不能依赖启停顺序； +- 某个设备 start 返回 false 时，当前实现会继续启动其他设备且不会回滚； +- DeviceManager 不捕获设备 init/start/stop 抛出的异常，后端应把预期失败转换为返回值，不能让异常越过 manager 边界； +- collection 配置要求 manager entry ID 能找到同 ID 子配置； +- ID 重复、不匹配或配置路径为空都会拒绝创建； +- `registerDevice()` 不会替调用方调用 `init()`； +- 运行阶段应把设备集合视为只读，动态注册必须在 service/task 启动前完成； +- `getDevice()` 类型不匹配或 ID 不存在时返回空指针。 + +不要在仍有 service/task 持有 manager 引用时调用 `destroyInstance()`。 + +## TaskManager + +完整任务实现指南见 [`../task/README.md`](../task/README.md)。 + +### PERIODIC_STEP + +- `control_period_s` 必须是有限正数； +- 所有周期任务的 `step()` 串行运行在同一个 scheduler 线程； +- 慢 I/O 或长计算会延迟其他周期任务； +- 只有 state 为 `RUNNING` 的任务会执行 `step()`； +- `step()` 内不能同步调用 `stopRunTask()` 等待当前 scheduler 自身。 + +### BLOCKING_SERVICE + +- TaskManager 只调用 `init/start/stop`，不调用 `step()`； +- `start()` 仍必须快速返回，由任务自己持有服务线程或事件循环； +- `stop()` 必须唤醒阻塞操作并 join 自己创建的线程。 + +### 启停语义 + +- manager 构造阶段只调用 task `init()`； +- 任一 task `start()` 失败，会停止此前已启动的任务且不启动 scheduler； +- 返回 false 的 task 必须自行清理本次 start 已经创建的部分资源，TaskManager 不会再调用该失败 task 的 stop； +- TaskManager 处于 running 状态时，停止流程先停止并 join scheduler，再调用每个 task 的 `stop()`； +- 未启动或已经停止时，`stopRunTask()` 会直接返回，不会再次逐个调用 task stop； +- 任务存储在 `unordered_map`，启动和停止顺序不确定； +- 有顺序依赖的工作应放入同一协调任务或显式建模； +- task 返回后，其内部状态并发安全由具体实现负责。 + +## MediaSourceHub + +关键文件： + +- [`media_source_hub/include/media_source_hub.h`](media_source_hub/include/media_source_hub.h) +- [`media_source_hub/src/device_media_source_adapter.cpp`](media_source_hub/src/device_media_source_adapter.cpp) +- [`../common/media/media_frame.h`](../common/media/media_frame.h) +- [`../common/base/ring_buffer.h`](../common/base/ring_buffer.h) + +当前默认轨道： + +| 来源 | Track ID | 默认 ring capacity | +| --- | --- | ---: | +| 摄像头彩色流 | `/video/color` | 64 | +| 麦克风主流 | `/audio/main` | 256 | + +当前 gRPC RGB/麦克风流和 QUIC 彩色/麦克风轨道使用 Hub；gRPC Depth/RGBD 仍直接读取设备帧。 + +### 注册新媒体源 + +1. 创建不可变 `TrackDescriptor`； +2. 使用唯一且稳定的 `track_id`； +3. 提供 `start(sink, cancelled)`； +4. 提供同步 `stop()`； +5. 视频源按需提供 `request_key_frame()`； +6. 调用 `registerSource()`； +7. 生产不可变 `MediaFramePtr`； +8. 保证 frame descriptor ID 与注册轨道一致； +9. 在重启、编码变化和中断时更新 producer 元数据和 discontinuity，由 adapter 更新 descriptor。 + +Descriptor 最低要求： + +- `id`、`source_id` 非空； +- kind 不能是 `UNKNOWN`； +- time base 分子和分母均大于零； +- generation 大于零。 + +### Source callback 生命周期 + +- 第一位 subscriber 触发一次 source start； +- 多位 subscriber 共享同一个采集生产者； +- 最后一份 Subscription reset/析构时同步调用 source stop； +- Hub 会先 close ring 并唤醒消费者；stop callback 必须解除生产端阻塞、停止采集并 join producer，形成同步发布屏障； +- start 必须在阻塞阶段检查 cancellation； +- cancellation predicate 必须快速、非阻塞，不能回调同一个 Hub； +- active source 不能 unregister，应先销毁全部 subscriptions； +- request-key-frame 与 stop 串行化，只在 source 运行时调用； +- adapter 的最后 lease 只停止媒体 streaming，不等于设备级 stop。 + +永久不响应 cancellation 的 start 虽会被隔离以避免 use-after-free，仍可能泄漏线程和外部资源，不能依赖该隔离替代正确实现。 + +### Subscription 与广播环形队列 + +- 每个消费线程单独调用一次 `subscribe()`； +- Subscription move-only 且为单消费者对象； +- 同一 Subscription 不能跨线程并发 read/reset； +- 满队列覆盖最旧帧，不阻塞生产者； +- `dropped_since_last_read` 是该消费者实际错过的帧数； +- ring 全局 dropped count 与 cursor dropped count 含义不同； +- `NEXT_PUBLISHED` 忽略已有帧； +- `OLDEST_AVAILABLE` 从最旧保留帧开始； +- `LATEST_AVAILABLE` 读取当前最新帧； +- source 重启会 reset ring、提升 `BroadcastFrameRing` 内部 generation，并把 ring 的 `ReadResult.sequence` 从 0 重新计数； +- close 唤醒等待者并拒绝新 publish。 + +ring generation 不等于 `TrackDescriptor::generation`，ring 的 `ReadResult.sequence` 也不等于 `MediaFrame::sequence`。Descriptor generation 和媒体帧 sequence 仍由 producer/adapter 维护。 + +协议消费者需要分别处理 ring `generation_changed`、descriptor generation、消费者 drop 和 frame discontinuity；任一不连续发生时都应传播状态，帧间编码还应请求关键帧。 + +## 新增第四种 Manager + +1. 先确认能力不是 DeviceManager、TaskManager 或 MediaSourceHub 的子职责； +2. 定义所有权、初始化、start/stop 和线程模型； +3. 避免新增无必要的全局单例； +4. 新建独立目录、头文件、实现和 CMake target； +5. 在 [`../CMakeLists.txt`](../CMakeLists.txt) 增加子目录； +6. 只在 `main.cpp` 或明确的上层 owner 组装； +7. 增加无设备生命周期、失败回滚和并发测试。 + +## 测试 + +MediaSourceHub： + +```bash +cmake --build build --target media_source_hub_test +ctest \ + --test-dir build \ + -R '^media_source_hub_test$' \ + --output-on-failure +``` + +DeviceManager 和 TaskManager 当前没有独立 CTest，这是现有测试缺口。修改其行为时至少补充： + +- fake device 创建、ID 冲突和 init/start/stop 失败； +- 设备依赖顺序； +- 周期任务调度和慢 step； +- service task 启停、重复 stop 和启动失败回滚； +- 并发查询、取消和 shutdown。 + +## 提交检查 + +- [ ] manager 没有包含厂商 wire protocol +- [ ] 初始化、start、stop 和 destroy 语义明确 +- [ ] 不依赖 unordered_map 的遍历顺序 +- [ ] 动态集合修改不会与查询并发 +- [ ] Media Subscription 每线程独立 +- [ ] stop 能唤醒并 join 所有工作线程 +- [ ] 新 manager 已加入 `cmvr-es/CMakeLists.txt` +- [ ] 无设备失败路径有测试 diff --git a/cmvr-es/manager/media_source_hub/CMakeLists.txt b/cmvr-es/manager/media_source_hub/CMakeLists.txt new file mode 100644 index 00000000..6a3634fd --- /dev/null +++ b/cmvr-es/manager/media_source_hub/CMakeLists.txt @@ -0,0 +1,61 @@ +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + cmake_minimum_required(VERSION 3.22) + project(cmvr_media_source_hub LANGUAGES CXX) + enable_testing() +endif() + +add_library(media_source_hub STATIC + src/media_source_hub.cpp +) + +target_compile_features(media_source_hub PUBLIC cxx_std_17) +target_include_directories(media_source_hub + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../.. +) + +add_library(cmvr_es::media_source_hub ALIAS media_source_hub) + +if(NOT CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + add_library(device_media_source_adapter STATIC + src/device_media_source_adapter.cpp + ) + target_compile_features(device_media_source_adapter PUBLIC cxx_std_17) + target_include_directories(device_media_source_adapter + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../.. + ) + target_link_libraries(device_media_source_adapter + PUBLIC + cmvr_es::media_source_hub + cmvr_es::common + cmvr_es::proto + cmvr_es::logging + ) + add_library(cmvr_es::device_media_source_adapter ALIAS device_media_source_adapter) +endif() + +option(CMVR_MEDIA_SOURCE_HUB_BUILD_TESTS + "Build the standalone MediaSourceHub self-test" + ${PROJECT_IS_TOP_LEVEL}) + +if(CMVR_MEDIA_SOURCE_HUB_BUILD_TESTS) + find_package(Threads REQUIRED) + add_executable(media_source_hub_test + tests/media_source_hub_test.cpp + ) + target_compile_features(media_source_hub_test PRIVATE cxx_std_17) + target_link_libraries(media_source_hub_test + PRIVATE + cmvr_es::media_source_hub + Threads::Threads + ) + # This self-test only links the static Hub and pthreads. In the root build, + # the project-wide third-party RUNPATH can otherwise make the loader pick up + # a vendor libstdc++.so (for example from the AUBO SDK), even though the test + # has no dependency on that SDK. + set_target_properties(media_source_hub_test PROPERTIES + SKIP_BUILD_RPATH TRUE + ) + add_test(NAME media_source_hub_test COMMAND media_source_hub_test) +endif() diff --git a/cmvr-es/manager/media_source_hub/include/device_media_source_adapter.h b/cmvr-es/manager/media_source_hub/include/device_media_source_adapter.h new file mode 100644 index 00000000..728e2d85 --- /dev/null +++ b/cmvr-es/manager/media_source_hub/include/device_media_source_adapter.h @@ -0,0 +1,38 @@ +#ifndef CMVR_ES_DEVICE_MEDIA_SOURCE_ADAPTER_H +#define CMVR_ES_DEVICE_MEDIA_SOURCE_ADAPTER_H + +#pragma once + +#include +#include +#include + +#include "devices/camera/abstract_camera.h" +#include "devices/microphone/abstract_microphone.h" +#include "manager/media_source_hub/include/media_source_hub.h" + +namespace cmvr::media { + +// Process-wide protocol-neutral media hub shared by gRPC and QUIC services. +MediaSourceHub& globalMediaSourceHub(); + +std::string cameraColorTrackId(const std::string& device_id); +std::string microphoneTrackId(const std::string& device_id); + +// Registration is idempotent for an already registered track. The adapter owns a +// short-lived pump thread and one startStreaming()/stopStreaming() lease only while +// at least one Hub subscription is active. It ensures start() succeeds but deliberately +// does not call stop(), because the base device lifecycle can also be owned by control RPCs. +bool ensureCameraMediaSource( + MediaSourceHub& hub, + const std::shared_ptr& camera, + size_t ring_capacity = 64); + +bool ensureMicrophoneMediaSource( + MediaSourceHub& hub, + const std::shared_ptr& microphone, + size_t ring_capacity = 256); + +} // namespace cmvr::media + +#endif // CMVR_ES_DEVICE_MEDIA_SOURCE_ADAPTER_H diff --git a/cmvr-es/manager/media_source_hub/include/media_source_hub.h b/cmvr-es/manager/media_source_hub/include/media_source_hub.h new file mode 100644 index 00000000..21dfc8f3 --- /dev/null +++ b/cmvr-es/manager/media_source_hub/include/media_source_hub.h @@ -0,0 +1,127 @@ +#ifndef CMVR_ES_MANAGER_MEDIA_SOURCE_HUB_H +#define CMVR_ES_MANAGER_MEDIA_SOURCE_HUB_H + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/base/ring_buffer.h" +#include "common/media/media_frame.h" + +namespace cmvr::media { + +// MediaSourceHub owns no protocol-specific state. A device or capture adapter registers +// start/stop callbacks and receives a sink callback when the first consumer subscribes. +class MediaSourceHub final { +public: + using FrameRing = BroadcastFrameRing; + using FrameReadResult = FrameRing::ReadResult; + using StartPosition = FrameRing::StartPosition; + using FrameSink = std::function; + // Cancellation checks run while MediaSourceHub protects source lifecycle + // state. Predicates must therefore be fast, non-blocking and must not call + // back into the same hub. + using CancelPredicate = std::function; + + struct SourceCallbacks { + // start() may run asynchronously. It must observe cancelled during any + // potentially blocking startup work and return false promptly once set. + // MediaSourceHub retains the callback state until a non-cooperative start + // eventually returns, so late completion cannot access destroyed state. + std::function start; + // stop() is the synchronous publication barrier for the last lease and + // must unblock and join the source producer before returning. + std::function stop; + std::function request_key_frame; + }; + +private: + struct SourceState; + +public: + class Subscription final { + public: + Subscription() = default; + ~Subscription(); + + Subscription(const Subscription&) = delete; + Subscription& operator=(const Subscription&) = delete; + Subscription(Subscription&& other) noexcept; + Subscription& operator=(Subscription&& other) noexcept; + + // A Subscription owns one reader cursor and is single-consumer. Moving, + // resetting, or reading the same object concurrently is unsupported; use + // one independent subscription per consumer thread. + bool valid() const; + explicit operator bool() const { return valid(); } + + // Returns the most recently observed immutable descriptor. A callback source may + // replace the initially registered UNKNOWN codec/config descriptor with the first + // real frame descriptor without invalidating existing subscriptions. + TrackDescriptorPtr descriptor() const; + + std::optional tryRead(); + std::optional waitRead(std::chrono::milliseconds timeout); + uint64_t droppedCount() const noexcept; + void reset(); + + private: + friend class MediaSourceHub; + Subscription(std::shared_ptr source, FrameRing::Cursor cursor); + + std::shared_ptr source_; + FrameRing::Cursor cursor_; + bool active_{false}; + }; + + MediaSourceHub(); + ~MediaSourceHub(); + + MediaSourceHub(const MediaSourceHub&) = delete; + MediaSourceHub& operator=(const MediaSourceHub&) = delete; + + bool registerSource( + TrackDescriptorPtr initial_descriptor, + SourceCallbacks callbacks, + size_t ring_capacity = 64); + + // Active sources cannot be unregistered. Destroy/reset their subscriptions first. + bool unregisterSource(const std::string& track_id); + + bool hasSource(const std::string& track_id) const; + std::vector listTracks() const; + size_t subscriberCount(const std::string& track_id) const; + + // Protocol adapters can request an IDR after a discontinuity without knowing the + // concrete camera implementation. Returns false when unsupported or not running. + bool requestKeyFrame(const std::string& track_id) const; + + Subscription subscribe( + const std::string& track_id, + StartPosition start_position = StartPosition::NEXT_PUBLISHED, + CancelPredicate cancelled = {}); + + // Stops all registered sources and invalidates outstanding subscriptions. The + // subscriptions remain destructible and their waitRead calls are awakened. + // A cooperative in-progress start is cancelled; a callback that violates the + // cancellation contract is quarantined with retained state rather than blocking + // shutdown or risking a use-after-free. + void shutdown(); + +private: + struct Impl; + std::shared_ptr impl_; +}; + +} // namespace cmvr::media + +#endif // CMVR_ES_MANAGER_MEDIA_SOURCE_HUB_H diff --git a/cmvr-es/manager/media_source_hub/src/device_media_source_adapter.cpp b/cmvr-es/manager/media_source_hub/src/device_media_source_adapter.cpp new file mode 100644 index 00000000..b1c4de7d --- /dev/null +++ b/cmvr-es/manager/media_source_hub/src/device_media_source_adapter.cpp @@ -0,0 +1,683 @@ +#include "manager/media_source_hub/include/device_media_source_adapter.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/base/logging/logger.h" + +namespace cmvr::media { +namespace { + +std::string normalizedCodec(std::string codec) { + codec.erase( + std::remove_if(codec.begin(), codec.end(), [](const unsigned char c) { + return !std::isalnum(c); + }), + codec.end()); + std::transform(codec.begin(), codec.end(), codec.begin(), [](const unsigned char c) { + return static_cast(std::tolower(c)); + }); + return codec; +} + +Codec videoCodec(const std::string& value) { + const std::string codec = normalizedCodec(value); + if (codec == "h264" || codec == "avc" || codec == "avc1" || codec == "libx264") { + return Codec::H264; + } + if (codec == "h265" || codec == "hevc" || codec == "hvc1" || codec == "libx265") { + return Codec::H265; + } + return Codec::UNKNOWN; +} + +PayloadFormat videoPayloadFormat( + const Codec codec, + const std::vector& payload) noexcept { + if (codec != Codec::H264 && codec != Codec::H265) { + return PayloadFormat::UNKNOWN; + } + const bool three_byte_start_code = payload.size() >= 3 && + payload[0] == 0U && payload[1] == 0U && payload[2] == 1U; + const bool four_byte_start_code = payload.size() >= 4 && + payload[0] == 0U && payload[1] == 0U && payload[2] == 0U && payload[3] == 1U; + return three_byte_start_code || four_byte_start_code + ? PayloadFormat::ANNEX_B + : PayloadFormat::UNKNOWN; +} + +Codec audioCodec(const std::string& value) { + const std::string codec = normalizedCodec(value); + if (codec == "opus") return Codec::OPUS; + if (codec == "aac") return Codec::AAC; + if (codec == "pcms16le") return Codec::PCM_S16LE; + return Codec::UNKNOWN; +} + +Codec audioCodec(const device::AudioStreamFrameData& source) { + const Codec codec = audioCodec(source.codec); + if (codec != Codec::UNKNOWN || !normalizedCodec(source.codec).empty()) { + return codec; + } + switch (source.format) { + case device::AudioStreamFormat::PCM: + return Codec::PCM_S16LE; + case device::AudioStreamFormat::AAC: + return Codec::AAC; + case device::AudioStreamFormat::OPUS: + return Codec::OPUS; + default: + return Codec::UNKNOWN; + } +} + +PayloadFormat audioPayloadFormat( + const Codec codec, + const std::vector& payload) noexcept { + switch (codec) { + case Codec::OPUS: + return PayloadFormat::OPUS_PACKET; + case Codec::PCM_S16LE: + return PayloadFormat::RAW; + case Codec::AAC: { + // FFmpeg encoders commonly expose raw AAC access units plus AudioSpecificConfig; + // only advertise ADTS when the sync word and layer bits are actually present. + const bool has_adts_header = payload.size() >= 2 && payload[0] == 0xFFU && + (payload[1] & 0xF6U) == 0xF0U; + return has_adts_header ? PayloadFormat::AAC_ADTS : PayloadFormat::RAW; + } + default: + return PayloadFormat::UNKNOWN; + } +} + +Rational sanitizedTimeBase( + const int32_t numerator, + const int32_t denominator, + const int32_t fallback_denominator) noexcept { + return Rational{ + numerator > 0 ? numerator : 1, + denominator > 0 ? denominator : std::max(1, fallback_denominator)}; +} + +uint64_t descriptorGeneration( + const uint64_t stream_epoch, + const uint32_t codec_generation) { + const uint64_t generation = (stream_epoch << 32U) | codec_generation; + return generation == 0 ? 1 : generation; +} + +template +struct PumpState : public std::enable_shared_from_this> { + explicit PumpState(std::shared_ptr device_ptr) + : device(std::move(device_ptr)) {} + + virtual ~PumpState() { + stop(); + } + + bool begin( + const MediaSourceHub::FrameSink& frame_sink, + const MediaSourceHub::CancelPredicate& cancelled) { + if (!frame_sink || !device) { + return false; + } + const auto cancellation_requested = [&cancelled] { + if (!cancelled) return false; + try { + return cancelled(); + } catch (...) { + return true; + } + }; + if (cancellation_requested()) { + return false; + } + + std::unique_lock lock(mutex); + if (running.load(std::memory_order_acquire)) { + return true; + } + if (worker.joinable()) { + // A previous worker must always be collected before a new capture lease starts. + std::thread stale_worker = std::move(worker); + lock.unlock(); + collectThread(std::move(stale_worker)); + lock.lock(); + } + + sink = frame_sink; + bool streaming_attempted = false; + try { + if (!device->start()) { + sink = {}; + return false; + } + if (cancellation_requested()) { + sink = {}; + return false; + } + streaming_attempted = true; + if (!device->startStreaming()) { + sink = {}; + lock.unlock(); + stopDeviceStreaming(); + return false; + } + streaming_started = true; + if (cancellation_requested()) { + streaming_started = false; + sink = {}; + lock.unlock(); + stopDeviceStreaming(); + return false; + } + running.store(true, std::memory_order_release); + try { + // The worker owns the pump while run() is active. This also + // makes the defensive self-stop/detach path lifetime-safe. + const auto self = this->shared_from_this(); + worker = std::thread([self] { self->run(); }); + } catch (...) { + running.store(false, std::memory_order_release); + streaming_started = false; + sink = {}; + lock.unlock(); + stopDeviceStreaming(); + return false; + } + } catch (const std::exception& error) { + CMVR_LOG(ERROR) << "[DeviceMediaSourceAdapter] Failed to start media source: " + << error.what(); + sink = {}; + lock.unlock(); + if (streaming_attempted) stopDeviceStreaming(); + return false; + } catch (...) { + CMVR_LOG(ERROR) << "[DeviceMediaSourceAdapter] Failed to start media source"; + sink = {}; + lock.unlock(); + if (streaming_attempted) stopDeviceStreaming(); + return false; + } + return true; + } + + void stop() noexcept { + std::thread thread; + bool stop_streaming = false; + { + std::lock_guard lock(mutex); + running.store(false, std::memory_order_release); + stop_streaming = streaming_started; + streaming_started = false; + sink = {}; + thread = std::move(worker); + } + if (stop_streaming) { + stopDeviceStreaming(); + } + if (thread.joinable()) { + collectThread(std::move(thread)); + } + } + + static void collectThread(std::thread thread) noexcept { + if (!thread.joinable()) { + return; + } + try { + if (thread.get_id() == std::this_thread::get_id()) { + thread.detach(); + } else { + thread.join(); + } + } catch (const std::exception& error) { + CMVR_LOG(ERROR) << "[DeviceMediaSourceAdapter] Failed to collect media pump: " + << error.what(); + if (thread.joinable()) { + try { + thread.detach(); + } catch (...) { + // std::thread's destructor would terminate if this extremely rare + // platform error occurred; there is no recoverable ownership path. + } + } + } + } + + virtual void run() = 0; + + void stopDeviceStreaming() noexcept { + try { + if (device) { + device->stopStreaming(); + } + } catch (const std::exception& error) { + CMVR_LOG(ERROR) << "[DeviceMediaSourceAdapter] Failed to stop media source: " + << error.what(); + } catch (...) { + CMVR_LOG(ERROR) << "[DeviceMediaSourceAdapter] Failed to stop media source"; + } + } + + std::shared_ptr device; + std::atomic running{false}; + std::mutex mutex; + std::thread worker; + MediaSourceHub::FrameSink sink; + bool streaming_started{false}; +}; + +struct CameraPump final : PumpState { + CameraPump(std::shared_ptr camera, std::string id) + : PumpState(std::move(camera)), track_id(std::move(id)) {} + ~CameraPump() override { stop(); } + + void run() override { + size_t cursor = 0; + uint64_t last_epoch = 0; + uint64_t last_sequence = 0; + uint64_t cached_config_generation = 0; + std::vector cached_codec_config; + TrackDescriptorPtr last_descriptor; + bool have_previous = false; + bool have_cached_config_generation = false; + bool waiting_for_key_frame = true; + bool pending_discontinuity = true; + bool requested_key_frame = false; + std::chrono::steady_clock::time_point last_key_frame_request; + + const auto request_key_frame = [&] { + const auto now = std::chrono::steady_clock::now(); + if (requested_key_frame && + now - last_key_frame_request < std::chrono::milliseconds(250)) { + return; + } + requested_key_frame = true; + last_key_frame_request = now; + try { + device->requestKeyFrame(); + } catch (...) { + // Unsupported/failed key-frame requests fall back to the encoder's GOP. + } + }; + request_key_frame(); + + while (running.load(std::memory_order_acquire)) { + device::StreamFrameData source; + try { + if (!device->waitEncodedFrame(source, cursor, std::chrono::milliseconds(50))) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } + } catch (const std::exception& error) { + CMVR_LOG(ERROR) << "[DeviceMediaSourceAdapter] Camera frame read failed for " + << track_id << ": " << error.what(); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + continue; + } catch (...) { + CMVR_LOG(ERROR) << "[DeviceMediaSourceAdapter] Camera frame read failed for " + << track_id; + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + continue; + } + if (source.rgbFrame.empty()) { + continue; + } + + const uint64_t descriptor_generation = descriptorGeneration( + source.stream_epoch, source.codec_config_generation); + if (!have_cached_config_generation || + cached_config_generation != descriptor_generation) { + cached_codec_config.clear(); + cached_config_generation = descriptor_generation; + have_cached_config_generation = true; + } + if (!source.codec_config.empty()) { + cached_codec_config = source.codec_config; + } + + TrackDescriptor::Config track; + track.id = track_id; + track.source_id = device->id(); + track.kind = MediaKind::VIDEO; + track.codec = videoCodec(source.codec); + track.payload_format = videoPayloadFormat(track.codec, source.rgbFrame); + track.time_base = sanitizedTimeBase( + source.time_base_num, + source.time_base_den, + source.fps); + track.width = static_cast(std::max(0, source.width)); + track.height = static_cast(std::max(0, source.height)); + track.nominal_rate = static_cast(std::max(0, source.fps)); + track.fx = source.intrinsics.fx; + track.fy = source.intrinsics.fy; + track.cx = source.intrinsics.cx; + track.cy = source.intrinsics.cy; + track.distortion.assign( + std::begin(source.intrinsics.coeffs), + std::end(source.intrinsics.coeffs)); + track.generation = descriptor_generation; + track.codec_config = cached_codec_config; + + const bool epoch_changed = have_previous && source.stream_epoch != last_epoch; + const bool sequence_wrapped = have_previous && + last_sequence == std::numeric_limits::max() && source.sequence != 0; + const bool sequence_gap = have_previous && !epoch_changed && + (sequence_wrapped || + (last_sequence != std::numeric_limits::max() && + source.sequence != last_sequence + 1)); + + TrackDescriptorPtr descriptor; + try { + descriptor = makeTrackDescriptor(std::move(track)); + } catch (const std::exception& error) { + CMVR_LOG(ERROR) << "[DeviceMediaSourceAdapter] Invalid camera descriptor for " + << track_id << ": " << error.what(); + pending_discontinuity = true; + last_epoch = source.stream_epoch; + last_sequence = source.sequence; + have_previous = true; + continue; + } + const bool descriptor_changed = last_descriptor && + !equivalentTrackDescriptor(*last_descriptor, *descriptor); + const bool discontinuity = !have_previous || source.discontinuity || epoch_changed || + sequence_gap || descriptor_changed; + const bool inter_frame_codec = descriptor->codec == Codec::H264 || + descriptor->codec == Codec::H265; + + if (discontinuity) { + pending_discontinuity = true; + if (inter_frame_codec) { + waiting_for_key_frame = true; + request_key_frame(); + } else { + waiting_for_key_frame = false; + } + } + + last_epoch = source.stream_epoch; + last_sequence = source.sequence; + last_descriptor = descriptor; + have_previous = true; + + if (waiting_for_key_frame && !source.bKey) { + request_key_frame(); + continue; + } + waiting_for_key_frame = false; + + MediaFrame::Config frame; + frame.descriptor = std::move(descriptor); + frame.payload = std::move(source.rgbFrame); + frame.sequence = source.sequence; + frame.pts = source.pts; + frame.dts = source.dts; + frame.duration = source.duration; + frame.capture_time_ns = source.capture_monotonic_ns > 0 + ? static_cast(source.capture_monotonic_ns) + : 0; + frame.capture_utc_ns = source.capture_utc_ns; + frame.key_frame = source.bKey; + frame.discontinuity = pending_discontinuity; + + MediaSourceHub::FrameSink current_sink; + { + std::lock_guard lock(mutex); + current_sink = sink; + } + if (current_sink) { + try { + current_sink(makeMediaFrame(std::move(frame))); + pending_discontinuity = false; + } catch (const std::exception& error) { + CMVR_LOG(ERROR) << "[DeviceMediaSourceAdapter] Invalid camera frame for " + << track_id << ": " << error.what(); + pending_discontinuity = true; + } + } + } + } + + std::string track_id; +}; + +struct MicrophonePump final : PumpState { + MicrophonePump(std::shared_ptr microphone, std::string id) + : PumpState(std::move(microphone)), track_id(std::move(id)) {} + ~MicrophonePump() override { stop(); } + + void run() override { + size_t cursor = 0; + uint64_t last_epoch = 0; + uint64_t last_sequence = 0; + uint64_t cached_config_generation = 0; + std::vector cached_codec_config; + TrackDescriptorPtr last_descriptor; + bool have_previous = false; + bool have_cached_config_generation = false; + bool pending_discontinuity = true; + while (running.load(std::memory_order_acquire)) { + device::AudioStreamFrameData source; + try { + if (!device->waitEncodedFrame(source, cursor, std::chrono::milliseconds(50))) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } + } catch (const std::exception& error) { + CMVR_LOG(ERROR) << "[DeviceMediaSourceAdapter] Microphone frame read failed for " + << track_id << ": " << error.what(); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + continue; + } catch (...) { + CMVR_LOG(ERROR) << "[DeviceMediaSourceAdapter] Microphone frame read failed for " + << track_id; + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + continue; + } + if (source.data.empty()) { + continue; + } + + const uint64_t descriptor_generation = descriptorGeneration( + source.stream_epoch, source.codec_config_generation); + if (!have_cached_config_generation || + cached_config_generation != descriptor_generation) { + cached_codec_config.clear(); + cached_config_generation = descriptor_generation; + have_cached_config_generation = true; + } + if (!source.codec_config.empty()) { + cached_codec_config = source.codec_config; + } + + TrackDescriptor::Config track; + track.id = track_id; + track.source_id = device->id(); + track.kind = MediaKind::AUDIO; + track.codec = audioCodec(source); + track.payload_format = audioPayloadFormat(track.codec, source.data); + track.time_base = sanitizedTimeBase( + source.time_base_num, + source.time_base_den, + source.sample_rate); + track.sample_rate = static_cast(std::max(0, source.sample_rate)); + track.channels = static_cast(std::max(0, source.channels)); + // Packet sample counts belong to MediaFrame::duration, not immutable track metadata. + track.nominal_rate = 0; + track.generation = descriptor_generation; + track.codec_config = cached_codec_config; + + const bool epoch_changed = have_previous && source.stream_epoch != last_epoch; + const bool sequence_wrapped = have_previous && + last_sequence == std::numeric_limits::max() && source.sequence != 0; + const bool sequence_gap = have_previous && !epoch_changed && + (sequence_wrapped || + (last_sequence != std::numeric_limits::max() && + source.sequence != last_sequence + 1)); + + TrackDescriptorPtr descriptor; + try { + descriptor = makeTrackDescriptor(std::move(track)); + } catch (const std::exception& error) { + CMVR_LOG(ERROR) << "[DeviceMediaSourceAdapter] Invalid microphone descriptor for " + << track_id << ": " << error.what(); + pending_discontinuity = true; + last_epoch = source.stream_epoch; + last_sequence = source.sequence; + have_previous = true; + continue; + } + const bool descriptor_changed = last_descriptor && + !equivalentTrackDescriptor(*last_descriptor, *descriptor); + if (!have_previous || source.discontinuity || epoch_changed || sequence_gap || + descriptor_changed) { + pending_discontinuity = true; + } + + last_epoch = source.stream_epoch; + last_sequence = source.sequence; + last_descriptor = descriptor; + have_previous = true; + + MediaFrame::Config frame; + frame.descriptor = std::move(descriptor); + frame.payload = std::move(source.data); + frame.sequence = source.sequence; + frame.pts = source.pts; + frame.dts = source.dts; + frame.duration = source.duration > 0 ? source.duration : source.nb_samples; + frame.capture_time_ns = source.capture_monotonic_ns > 0 + ? static_cast(source.capture_monotonic_ns) + : 0; + frame.capture_utc_ns = source.capture_utc_ns; + frame.key_frame = true; + frame.discontinuity = pending_discontinuity; + + MediaSourceHub::FrameSink current_sink; + { + std::lock_guard lock(mutex); + current_sink = sink; + } + if (current_sink) { + try { + current_sink(makeMediaFrame(std::move(frame))); + pending_discontinuity = false; + } catch (const std::exception& error) { + CMVR_LOG(ERROR) << "[DeviceMediaSourceAdapter] Invalid microphone frame for " + << track_id << ": " << error.what(); + pending_discontinuity = true; + } + } + } + } + + std::string track_id; +}; + +TrackDescriptorPtr initialTrack( + std::string track_id, + std::string source_id, + const MediaKind kind) { + TrackDescriptor::Config config; + config.id = std::move(track_id); + config.source_id = std::move(source_id); + config.kind = kind; + // A placeholder descriptor is never emitted as a media sample, but it + // still carries a mathematically valid neutral time base. + config.time_base = Rational{1, 1}; + config.generation = 1; + return makeTrackDescriptor(std::move(config)); +} + +} // namespace + +MediaSourceHub& globalMediaSourceHub() { + static MediaSourceHub hub; + return hub; +} + +std::string cameraColorTrackId(const std::string& device_id) { + return device_id + "/video/color"; +} + +std::string microphoneTrackId(const std::string& device_id) { + return device_id + "/audio/main"; +} + +bool ensureCameraMediaSource( + MediaSourceHub& hub, + const std::shared_ptr& camera, + const size_t ring_capacity) { + if (!camera || camera->id().empty()) { + return false; + } + const std::string track_id = cameraColorTrackId(camera->id()); + if (hub.hasSource(track_id)) { + return true; + } + + const auto pump = std::make_shared(camera, track_id); + MediaSourceHub::SourceCallbacks callbacks; + callbacks.start = [pump]( + const MediaSourceHub::FrameSink& sink, + const MediaSourceHub::CancelPredicate& cancelled) { + return pump->begin(sink, cancelled); + }; + callbacks.stop = [pump] { pump->stop(); }; + callbacks.request_key_frame = [camera] { return camera->requestKeyFrame(); }; + const bool registered = hub.registerSource( + initialTrack(track_id, camera->id(), MediaKind::VIDEO), + std::move(callbacks), + ring_capacity); + if (!registered && !hub.hasSource(track_id)) { + CMVR_LOG(ERROR) << "[DeviceMediaSourceAdapter] Failed to register camera track: " << track_id; + return false; + } + return true; +} + +bool ensureMicrophoneMediaSource( + MediaSourceHub& hub, + const std::shared_ptr& microphone, + const size_t ring_capacity) { + if (!microphone || microphone->id().empty()) { + return false; + } + const std::string track_id = microphoneTrackId(microphone->id()); + if (hub.hasSource(track_id)) { + return true; + } + + const auto pump = std::make_shared(microphone, track_id); + MediaSourceHub::SourceCallbacks callbacks; + callbacks.start = [pump]( + const MediaSourceHub::FrameSink& sink, + const MediaSourceHub::CancelPredicate& cancelled) { + return pump->begin(sink, cancelled); + }; + callbacks.stop = [pump] { pump->stop(); }; + const bool registered = hub.registerSource( + initialTrack(track_id, microphone->id(), MediaKind::AUDIO), + std::move(callbacks), + ring_capacity); + if (!registered && !hub.hasSource(track_id)) { + CMVR_LOG(ERROR) << "[DeviceMediaSourceAdapter] Failed to register microphone track: " << track_id; + return false; + } + return true; +} + +} // namespace cmvr::media diff --git a/cmvr-es/manager/media_source_hub/src/media_source_hub.cpp b/cmvr-es/manager/media_source_hub/src/media_source_hub.cpp new file mode 100644 index 00000000..6eebab28 --- /dev/null +++ b/cmvr-es/manager/media_source_hub/src/media_source_hub.cpp @@ -0,0 +1,588 @@ +#include "manager/media_source_hub/include/media_source_hub.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace cmvr::media { + +struct MediaSourceHub::SourceState final : public std::enable_shared_from_this { + enum class Lifecycle { + STOPPED, + STARTING, + RUNNING, + STOPPING + }; + + struct StartAttempt { + size_t waiters{0}; + bool completed{false}; + bool succeeded{false}; + std::atomic cancel_requested{false}; + }; + + SourceState( + TrackDescriptorPtr initial_descriptor, + SourceCallbacks source_callbacks, + const size_t ring_capacity) + : track_id(initial_descriptor->id), + descriptor(std::move(initial_descriptor)), + callbacks(std::move(source_callbacks)), + ring(ring_capacity) {} + + FrameSink makeSink() { + const std::weak_ptr weak_source = shared_from_this(); + return [weak_source](MediaFramePtr frame) { + if (const auto source = weak_source.lock()) { + source->acceptFrame(std::move(frame)); + } + }; + } + + void acceptFrame(MediaFramePtr frame) { + if (!frame || !frame->descriptor || frame->descriptor->id != track_id) { + return; + } + + { + std::lock_guard lock(lifecycle_mutex); + if (!registered || + (lifecycle != Lifecycle::STARTING && lifecycle != Lifecycle::RUNNING)) { + return; + } + } + + const TrackDescriptorPtr current = std::atomic_load(&descriptor); + if (!current || !equivalentTrackDescriptor(*current, *frame->descriptor)) { + // C++17 atomic shared_ptr free functions provide an atomic descriptor snapshot to + // all subscriptions while frames remain immutable. + std::atomic_store(&descriptor, frame->descriptor); + } + ring.publish(std::move(frame)); + } + + static bool isCancelled(const CancelPredicate& cancelled) noexcept { + if (!cancelled) return false; + try { + return cancelled(); + } catch (...) { + return true; + } + } + + bool invokeStart( + const FrameSink& sink, + const CancelPredicate& cancelled) noexcept { + std::lock_guard callback_lock(callback_mutex); + try { + return callbacks.start && callbacks.start(sink, cancelled); + } catch (...) { + return false; + } + } + + void invokeStop() noexcept { + std::lock_guard callback_lock(callback_mutex); + try { + if (callbacks.stop) callbacks.stop(); + } catch (...) { + } + } + + void completeStart( + const std::shared_ptr& attempt, + const bool started) { + bool stop_abandoned_start = false; + { + std::lock_guard lock(lifecycle_mutex); + if (start_attempt != attempt || lifecycle != Lifecycle::STARTING) { + return; + } + attempt->completed = true; + attempt->succeeded = started; + if (started && registered && attempt->waiters != 0U) { + lifecycle = Lifecycle::RUNNING; + } else if (started) { + lifecycle = Lifecycle::STOPPING; + ring.close(); + stop_abandoned_start = true; + } else { + lifecycle = Lifecycle::STOPPED; + } + lifecycle_condition.notify_all(); + } + + if (stop_abandoned_start) { + invokeStop(); + std::lock_guard lock(lifecycle_mutex); + if (lifecycle == Lifecycle::STOPPING) { + lifecycle = Lifecycle::STOPPED; + } + lifecycle_condition.notify_all(); + } + } + + bool acquire( + const StartPosition start_position, + FrameRing::Cursor& cursor, + const CancelPredicate& cancelled) { + std::unique_lock lock(lifecycle_mutex); + while (lifecycle == Lifecycle::STOPPING) { + if (!registered || isCancelled(cancelled)) return false; + lifecycle_condition.wait_for(lock, std::chrono::milliseconds(10)); + } + if (!registered || isCancelled(cancelled)) return false; + + if (lifecycle == Lifecycle::RUNNING) { + ++subscriber_count; + lock.unlock(); + cursor = ring.makeCursor(start_position); + return true; + } + + std::shared_ptr attempt; + if (lifecycle == Lifecycle::STOPPED) { + lifecycle = Lifecycle::STARTING; + ring.reset(); + const FrameSink sink = makeSink(); + attempt = std::make_shared(); + attempt->waiters = 1U; + start_attempt = attempt; + const auto self = shared_from_this(); + try { + std::thread([self, attempt, sink]() { + const CancelPredicate cancelled = [attempt] { + return attempt->cancel_requested.load( + std::memory_order_acquire); + }; + const bool started = self->invokeStart(sink, cancelled); + self->completeStart(attempt, started); + }).detach(); + } catch (...) { + start_attempt.reset(); + lifecycle = Lifecycle::STOPPED; + lifecycle_condition.notify_all(); + return false; + } + } else if (lifecycle == Lifecycle::STARTING) { + attempt = start_attempt; + if (!attempt || attempt->cancel_requested.load(std::memory_order_acquire)) { + return false; + } + ++attempt->waiters; + } else { + return false; + } + + while (registered && !attempt->completed) { + if (isCancelled(cancelled)) { + if (attempt->waiters != 0U) --attempt->waiters; + if (attempt->waiters == 0U) { + attempt->cancel_requested.store(true, std::memory_order_release); + } + lifecycle_condition.notify_all(); + return false; + } + lifecycle_condition.wait_for(lock, std::chrono::milliseconds(10)); + } + + const bool caller_cancelled = isCancelled(cancelled); + const bool acquired = !caller_cancelled && registered && attempt->completed && + attempt->succeeded && + lifecycle == Lifecycle::RUNNING; + if (attempt->waiters != 0U) --attempt->waiters; + if (!acquired) { + if (attempt->waiters == 0U) { + attempt->cancel_requested.store(true, std::memory_order_release); + } + const bool stop_unclaimed_source = + start_attempt == attempt && attempt->waiters == 0U && + subscriber_count == 0U && + lifecycle == Lifecycle::RUNNING; + if (stop_unclaimed_source) { + lifecycle = Lifecycle::STOPPING; + ring.close(); + lock.unlock(); + invokeStop(); + lock.lock(); + if (lifecycle == Lifecycle::STOPPING) { + lifecycle = Lifecycle::STOPPED; + } + lifecycle_condition.notify_all(); + } + return false; + } + ++subscriber_count; + lock.unlock(); + cursor = ring.makeCursor(start_position); + return true; + } + + void release() { + std::unique_lock lock(lifecycle_mutex); + if (subscriber_count == 0) { + return; + } + --subscriber_count; + if (subscriber_count != 0 || lifecycle != Lifecycle::RUNNING) { + return; + } + + lifecycle = Lifecycle::STOPPING; + ring.close(); + lock.unlock(); + invokeStop(); + lock.lock(); + lifecycle = Lifecycle::STOPPED; + lifecycle_condition.notify_all(); + } + + bool deactivateIfUnused() { + std::unique_lock lock(lifecycle_mutex); + if (subscriber_count != 0) { + return false; + } + registered = false; + ring.close(); + if (lifecycle == Lifecycle::STARTING) { + if (start_attempt) { + start_attempt->cancel_requested.store(true, std::memory_order_release); + } + lifecycle_condition.notify_all(); + return true; + } + if (lifecycle == Lifecycle::STOPPING || lifecycle == Lifecycle::STOPPED) { + lifecycle_condition.notify_all(); + return true; + } + + lifecycle = Lifecycle::STOPPING; + lock.unlock(); + invokeStop(); + lock.lock(); + if (lifecycle == Lifecycle::STOPPING) { + lifecycle = Lifecycle::STOPPED; + } + lifecycle_condition.notify_all(); + return true; + } + + void shutdown() { + std::unique_lock lock(lifecycle_mutex); + registered = false; + ring.close(); + if (lifecycle == Lifecycle::STARTING) { + if (start_attempt) { + start_attempt->cancel_requested.store(true, std::memory_order_release); + } + lifecycle_condition.notify_all(); + return; + } + if (lifecycle == Lifecycle::STOPPING) { + lifecycle_condition.notify_all(); + return; + } + if (lifecycle == Lifecycle::STOPPED) { + lifecycle_condition.notify_all(); + return; + } + + lifecycle = Lifecycle::STOPPING; + lock.unlock(); + invokeStop(); + lock.lock(); + if (lifecycle == Lifecycle::STOPPING) { + lifecycle = Lifecycle::STOPPED; + } + lifecycle_condition.notify_all(); + } + + bool validForSubscription() const { + std::lock_guard lock(lifecycle_mutex); + return registered && lifecycle == Lifecycle::RUNNING; + } + + size_t subscriberCount() const { + std::lock_guard lock(lifecycle_mutex); + return subscriber_count; + } + + bool requestKeyFrame() const { + // Serialize with stop first, then re-check lifecycle. A stop that has + // already begun rejects the request; a stop that begins afterwards + // waits for this callback before invoking the device stop barrier. + std::lock_guard callback_lock(callback_mutex); + { + std::lock_guard lock(lifecycle_mutex); + if (!registered || lifecycle != Lifecycle::RUNNING || !callbacks.request_key_frame) { + return false; + } + } + try { + return callbacks.request_key_frame(); + } catch (...) { + return false; + } + } + + TrackDescriptorPtr currentDescriptor() const { + return std::atomic_load(&descriptor); + } + + const std::string track_id; + mutable TrackDescriptorPtr descriptor; + const SourceCallbacks callbacks; + FrameRing ring; + + mutable std::mutex callback_mutex; + mutable std::mutex lifecycle_mutex; + std::condition_variable lifecycle_condition; + Lifecycle lifecycle{Lifecycle::STOPPED}; + size_t subscriber_count{0}; + bool registered{true}; + std::shared_ptr start_attempt; +}; + +struct MediaSourceHub::Impl final { + mutable std::mutex mutex; + std::unordered_map> sources; +}; + +MediaSourceHub::Subscription::Subscription( + std::shared_ptr source, + FrameRing::Cursor cursor) + : source_(std::move(source)), + cursor_(std::move(cursor)), + active_(static_cast(source_)) {} + +MediaSourceHub::Subscription::~Subscription() { + reset(); +} + +MediaSourceHub::Subscription::Subscription(Subscription&& other) noexcept + : source_(std::move(other.source_)), + cursor_(other.cursor_), + active_(other.active_) { + other.active_ = false; +} + +MediaSourceHub::Subscription& MediaSourceHub::Subscription::operator=(Subscription&& other) noexcept { + if (this == &other) { + return *this; + } + reset(); + source_ = std::move(other.source_); + cursor_ = other.cursor_; + active_ = other.active_; + other.active_ = false; + return *this; +} + +bool MediaSourceHub::Subscription::valid() const { + return active_ && source_ && source_->validForSubscription(); +} + +TrackDescriptorPtr MediaSourceHub::Subscription::descriptor() const { + return source_ ? source_->currentDescriptor() : nullptr; +} + +std::optional MediaSourceHub::Subscription::tryRead() { + if (!active_ || !source_) { + return std::nullopt; + } + return source_->ring.tryRead(cursor_); +} + +std::optional MediaSourceHub::Subscription::waitRead( + const std::chrono::milliseconds timeout) { + if (!active_ || !source_) { + return std::nullopt; + } + return source_->ring.waitRead(cursor_, timeout); +} + +uint64_t MediaSourceHub::Subscription::droppedCount() const noexcept { + return cursor_.dropped_count; +} + +void MediaSourceHub::Subscription::reset() { + if (active_ && source_) { + source_->release(); + } + active_ = false; + source_.reset(); +} + +MediaSourceHub::MediaSourceHub() + : impl_(std::make_shared()) {} + +MediaSourceHub::~MediaSourceHub() { + shutdown(); +} + +bool MediaSourceHub::registerSource( + TrackDescriptorPtr initial_descriptor, + SourceCallbacks callbacks, + const size_t ring_capacity) { + if (!impl_ || !initial_descriptor || initial_descriptor->id.empty() || + !callbacks.start || ring_capacity == 0) { + return false; + } + + std::shared_ptr source; + try { + source = std::make_shared( + std::move(initial_descriptor), std::move(callbacks), ring_capacity); + } catch (...) { + return false; + } + + std::lock_guard lock(impl_->mutex); + return impl_->sources.emplace(source->track_id, std::move(source)).second; +} + +bool MediaSourceHub::unregisterSource(const std::string& track_id) { + if (!impl_ || track_id.empty()) { + return false; + } + + std::shared_ptr source; + { + std::lock_guard lock(impl_->mutex); + const auto it = impl_->sources.find(track_id); + if (it == impl_->sources.end()) { + return false; + } + source = it->second; + } + + if (!source->deactivateIfUnused()) { + return false; + } + + std::lock_guard lock(impl_->mutex); + const auto it = impl_->sources.find(track_id); + if (it != impl_->sources.end() && it->second == source) { + impl_->sources.erase(it); + return true; + } + return false; +} + +bool MediaSourceHub::hasSource(const std::string& track_id) const { + if (!impl_) { + return false; + } + std::lock_guard lock(impl_->mutex); + return impl_->sources.find(track_id) != impl_->sources.end(); +} + +std::vector MediaSourceHub::listTracks() const { + std::vector> sources; + if (!impl_) { + return {}; + } + { + std::lock_guard lock(impl_->mutex); + sources.reserve(impl_->sources.size()); + for (const auto& [track_id, source] : impl_->sources) { + (void)track_id; + sources.push_back(source); + } + } + + std::vector descriptors; + descriptors.reserve(sources.size()); + for (const auto& source : sources) { + descriptors.push_back(source->currentDescriptor()); + } + std::sort(descriptors.begin(), descriptors.end(), [](const auto& lhs, const auto& rhs) { + if (!lhs) return static_cast(rhs); + if (!rhs) return false; + return lhs->id < rhs->id; + }); + return descriptors; +} + +size_t MediaSourceHub::subscriberCount(const std::string& track_id) const { + if (!impl_) { + return 0; + } + std::shared_ptr source; + { + std::lock_guard lock(impl_->mutex); + const auto it = impl_->sources.find(track_id); + if (it == impl_->sources.end()) { + return 0; + } + source = it->second; + } + return source->subscriberCount(); +} + +bool MediaSourceHub::requestKeyFrame(const std::string& track_id) const { + if (!impl_) { + return false; + } + std::shared_ptr source; + { + std::lock_guard lock(impl_->mutex); + const auto it = impl_->sources.find(track_id); + if (it == impl_->sources.end()) { + return false; + } + source = it->second; + } + return source->requestKeyFrame(); +} + +MediaSourceHub::Subscription MediaSourceHub::subscribe( + const std::string& track_id, + const StartPosition start_position, + CancelPredicate cancelled) { + if (!impl_) { + return {}; + } + + std::shared_ptr source; + { + std::lock_guard lock(impl_->mutex); + const auto it = impl_->sources.find(track_id); + if (it == impl_->sources.end()) { + return {}; + } + source = it->second; + } + + FrameRing::Cursor cursor; + if (!source->acquire(start_position, cursor, cancelled)) { + return {}; + } + return Subscription(std::move(source), std::move(cursor)); +} + +void MediaSourceHub::shutdown() { + if (!impl_) { + return; + } + + std::vector> sources; + { + std::lock_guard lock(impl_->mutex); + sources.reserve(impl_->sources.size()); + for (auto& [track_id, source] : impl_->sources) { + (void)track_id; + sources.push_back(std::move(source)); + } + impl_->sources.clear(); + } + for (const auto& source : sources) { + source->shutdown(); + } +} + +} // namespace cmvr::media diff --git a/cmvr-es/manager/media_source_hub/tests/media_source_hub_test.cpp b/cmvr-es/manager/media_source_hub/tests/media_source_hub_test.cpp new file mode 100644 index 00000000..fdb60f73 --- /dev/null +++ b/cmvr-es/manager/media_source_hub/tests/media_source_hub_test.cpp @@ -0,0 +1,514 @@ +#include "manager/media_source_hub/include/media_source_hub.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace std::chrono_literals; +using cmvr::media::Codec; +using cmvr::media::MediaFrame; +using cmvr::media::MediaFramePtr; +using cmvr::media::MediaKind; +using cmvr::media::MediaSourceHub; +using cmvr::media::PayloadFormat; +using cmvr::media::Rational; +using cmvr::media::TrackDescriptor; +using cmvr::media::TrackDescriptorPtr; + +static_assert(!std::is_copy_assignable::value, "MediaFrame must be immutable"); +static_assert(!std::is_copy_assignable::value, "TrackDescriptor must be immutable"); +static_assert(std::is_same::value, + "MediaFramePtr must share const frames"); + +int failures = 0; + +#define CHECK_TRUE(expression) \ + do { \ + if (!(expression)) { \ + std::cerr << __FILE__ << ':' << __LINE__ << " check failed: " #expression << '\n'; \ + ++failures; \ + } \ + } while (false) + +TrackDescriptorPtr makeVideoDescriptor( + const Codec codec, + const uint64_t generation, + std::vector codec_config = {}) { + TrackDescriptor::Config config; + config.id = "camera.front.video"; + config.source_id = "camera.front"; + config.kind = MediaKind::VIDEO; + config.codec = codec; + config.payload_format = codec == Codec::UNKNOWN ? PayloadFormat::UNKNOWN : PayloadFormat::ANNEX_B; + config.time_base = Rational{1, 90000}; + config.width = 640; + config.height = 360; + config.nominal_rate = 30; + config.generation = generation; + config.codec_config = std::move(codec_config); + return cmvr::media::makeTrackDescriptor(std::move(config)); +} + +MediaFramePtr makeFrame( + TrackDescriptorPtr descriptor, + const uint64_t sequence, + const uint8_t marker) { + MediaFrame::Config config; + config.descriptor = std::move(descriptor); + config.payload = {marker, static_cast(marker + 1)}; + config.sequence = sequence; + config.pts = static_cast(sequence * 3000); + config.dts = config.pts; + config.duration = 3000; + config.capture_time_ns = sequence * 1000000; + config.capture_utc_ns = 1700000000000000000LL + static_cast(sequence); + config.key_frame = sequence == 0; + return cmvr::media::makeMediaFrame(std::move(config)); +} + +void testMediaMetadataValidation() { + TrackDescriptor::Config invalid; + invalid.id = "invalid.video"; + invalid.source_id = "invalid"; + invalid.kind = MediaKind::VIDEO; + invalid.time_base = Rational{0, 1}; + bool rejected = false; + try { + (void)cmvr::media::makeTrackDescriptor(std::move(invalid)); + } catch (const std::invalid_argument&) { + rejected = true; + } + CHECK_TRUE(rejected); + + const auto frame = makeFrame(makeVideoDescriptor(Codec::H264, 1), 7, 1); + CHECK_TRUE(frame->capture_time_ns == 7000000); + CHECK_TRUE(frame->capture_utc_ns == 1700000000000000007LL); + CHECK_TRUE(frame->duration == 3000); +} + +void testLegacySpmcCompatibility() { + bool ring_zero_capacity_rejected = false; + try { + RingBuffer invalid_ring(0); + } catch (const std::invalid_argument&) { + ring_zero_capacity_rejected = true; + } + CHECK_TRUE(ring_zero_capacity_rejected); + + bool spmc_zero_capacity_rejected = false; + try { + SPMCRingBuffer invalid_ring(0); + } catch (const std::invalid_argument&) { + spmc_zero_capacity_rejected = true; + } + CHECK_TRUE(spmc_zero_capacity_rejected); + + SPMCRingBuffer ring(2); + ring.push(10); + ring.push(20); + CHECK_TRUE(ring.size() == 2); + CHECK_TRUE(ring.getHead() == 2); + CHECK_TRUE(ring.getTail() == 0); + CHECK_TRUE(ring.getLast().has_value() && *ring.getLast() == 20); + + size_t reader = 0; + CHECK_TRUE(ring.pop(reader).has_value()); + ring.push(30); + ring.push(40); + CHECK_TRUE(!ring.pop(reader).has_value()); + CHECK_TRUE(reader == ring.getTail()); + CHECK_TRUE(ring.pop(reader).has_value()); + + ring.clear(); + CHECK_TRUE(ring.empty()); + CHECK_TRUE(ring.getHead() == 4); + ring.push(50); + CHECK_TRUE(!ring.pop(reader).has_value()); + const auto after_clear = ring.pop(reader); + CHECK_TRUE(after_clear.has_value() && *after_clear == 50); +} + +void testBroadcastFrameRing() { + using Ring = BroadcastFrameRing; + Ring ring(2); + const auto descriptor = makeVideoDescriptor(Codec::H264, 1, {1, 2, 3}); + auto cursor = ring.makeCursor(Ring::StartPosition::OLDEST_AVAILABLE); + + CHECK_TRUE(ring.publish(makeFrame(descriptor, 0, 10)).value() == 0); + CHECK_TRUE(ring.publish(makeFrame(descriptor, 1, 20)).value() == 1); + CHECK_TRUE(ring.publish(makeFrame(descriptor, 2, 30)).value() == 2); + + const auto first = ring.tryRead(cursor); + CHECK_TRUE(first.has_value()); + CHECK_TRUE(first->sequence == 1); + CHECK_TRUE(first->value->sequence == 1); + CHECK_TRUE(first->dropped_count == 1); + CHECK_TRUE(first->dropped_since_last_read == 1); + CHECK_TRUE(ring.stats().dropped_count == 1); + + const auto second = ring.tryRead(cursor); + CHECK_TRUE(second.has_value() && second->sequence == 2); + CHECK_TRUE(second->dropped_since_last_read == 0); + + auto waiting_cursor = ring.makeCursor(Ring::StartPosition::NEXT_PUBLISHED); + auto waiting_read = std::async(std::launch::async, [&ring, &waiting_cursor] { + return ring.waitRead(waiting_cursor, 1s); + }); + std::this_thread::sleep_for(10ms); + ring.publish(makeFrame(descriptor, 3, 40)); + CHECK_TRUE(waiting_read.wait_for(500ms) == std::future_status::ready); + CHECK_TRUE(waiting_read.get().has_value()); + + const uint64_t next_generation = ring.reset(); + CHECK_TRUE(next_generation == 2); + ring.publish(makeFrame(descriptor, 4, 50)); + const auto after_reset = ring.tryRead(cursor); + CHECK_TRUE(after_reset.has_value()); + CHECK_TRUE(after_reset->generation == 2); + CHECK_TRUE(after_reset->sequence == 0); + CHECK_TRUE(after_reset->generation_changed); + + auto close_cursor = ring.makeCursor(Ring::StartPosition::NEXT_PUBLISHED); + auto close_wait = std::async(std::launch::async, [&ring, &close_cursor] { + return ring.waitRead(close_cursor, 2s); + }); + ring.close(); + CHECK_TRUE(close_wait.wait_for(500ms) == std::future_status::ready); + CHECK_TRUE(!close_wait.get().has_value()); + CHECK_TRUE(!ring.publish(makeFrame(descriptor, 5, 60)).has_value()); +} + +void testBroadcastConcurrency() { + using Ring = BroadcastFrameRing; + constexpr uint64_t frame_count = 500; + Ring ring(frame_count); + const auto descriptor = makeVideoDescriptor(Codec::H264, 1, {1, 2, 3}); + auto first_cursor = ring.makeCursor(Ring::StartPosition::OLDEST_AVAILABLE); + auto second_cursor = ring.makeCursor(Ring::StartPosition::OLDEST_AVAILABLE); + + auto consume = [&ring](Ring::Cursor& cursor) { + uint64_t expected = 0; + while (expected < frame_count) { + const auto result = ring.waitRead(cursor, 1s); + if (!result || result->sequence != expected || result->value->sequence != expected) { + return false; + } + ++expected; + } + return cursor.dropped_count == 0; + }; + + auto first_consumer = std::async(std::launch::async, consume, std::ref(first_cursor)); + auto second_consumer = std::async(std::launch::async, consume, std::ref(second_cursor)); + std::thread producer([&ring, &descriptor] { + for (uint64_t sequence = 0; sequence < frame_count; ++sequence) { + ring.publish(makeFrame(descriptor, sequence, static_cast(sequence))); + } + }); + + producer.join(); + CHECK_TRUE(first_consumer.get()); + CHECK_TRUE(second_consumer.get()); +} + +void testHubLifecycleAndDescriptorRefresh() { + MediaSourceHub hub; + const auto initial_descriptor = makeVideoDescriptor(Codec::UNKNOWN, 1); + std::atomic start_count{0}; + std::atomic stop_count{0}; + std::atomic key_frame_requests{0}; + std::mutex sink_mutex; + MediaSourceHub::FrameSink sink; + + MediaSourceHub::SourceCallbacks callbacks; + callbacks.start = [&](const MediaSourceHub::FrameSink& callback_sink, + const MediaSourceHub::CancelPredicate&) { + { + std::lock_guard lock(sink_mutex); + sink = callback_sink; + } + ++start_count; + return true; + }; + callbacks.stop = [&] { + ++stop_count; + std::lock_guard lock(sink_mutex); + sink = {}; + }; + callbacks.request_key_frame = [&] { + ++key_frame_requests; + return true; + }; + + CHECK_TRUE(hub.registerSource(initial_descriptor, callbacks, 4)); + CHECK_TRUE(!hub.registerSource(initial_descriptor, callbacks, 4)); + CHECK_TRUE(hub.hasSource(initial_descriptor->id)); + CHECK_TRUE(hub.listTracks().size() == 1); + + auto first = hub.subscribe(initial_descriptor->id); + auto second = hub.subscribe(initial_descriptor->id); + CHECK_TRUE(first.valid() && second.valid()); + CHECK_TRUE(hub.requestKeyFrame(initial_descriptor->id)); + CHECK_TRUE(key_frame_requests == 1); + CHECK_TRUE(start_count == 1); + CHECK_TRUE(hub.subscriberCount(initial_descriptor->id) == 2); + CHECK_TRUE(first.descriptor()->codec == Codec::UNKNOWN); + CHECK_TRUE(!hub.unregisterSource(initial_descriptor->id)); + + // Content changes at the same generation must atomically replace the initial descriptor. + const auto actual_descriptor = makeVideoDescriptor(Codec::H264, 1, {0, 0, 0, 1, 0x67}); + MediaSourceHub::FrameSink producer; + { + std::lock_guard lock(sink_mutex); + producer = sink; + } + CHECK_TRUE(static_cast(producer)); + const auto shared_frame = makeFrame(actual_descriptor, 0, 70); + producer(shared_frame); + + const auto first_read = first.waitRead(500ms); + const auto second_read = second.waitRead(500ms); + CHECK_TRUE(first_read.has_value() && first_read->value == shared_frame); + CHECK_TRUE(second_read.has_value() && second_read->value == shared_frame); + CHECK_TRUE(first.descriptor() == actual_descriptor); + CHECK_TRUE(second.descriptor()->codec_config == actual_descriptor->codec_config); + + first.reset(); + CHECK_TRUE(stop_count == 0); + CHECK_TRUE(hub.subscriberCount(initial_descriptor->id) == 1); + second.reset(); + CHECK_TRUE(stop_count == 1); + CHECK_TRUE(!hub.requestKeyFrame(initial_descriptor->id)); + CHECK_TRUE(hub.subscriberCount(initial_descriptor->id) == 0); + + { + auto restarted = hub.subscribe(initial_descriptor->id); + CHECK_TRUE(restarted.valid()); + CHECK_TRUE(start_count == 2); + } + CHECK_TRUE(stop_count == 2); + CHECK_TRUE(hub.unregisterSource(initial_descriptor->id)); + CHECK_TRUE(!hub.hasSource(initial_descriptor->id)); +} + +void testHubFailedStartAndShutdown() { + MediaSourceHub hub; + const auto descriptor = makeVideoDescriptor(Codec::UNKNOWN, 1); + std::atomic start_attempts{0}; + std::atomic retry_stop_count{0}; + MediaSourceHub::SourceCallbacks failed_callbacks; + failed_callbacks.start = [&](const MediaSourceHub::FrameSink&, + const MediaSourceHub::CancelPredicate&) { + return ++start_attempts >= 2; + }; + failed_callbacks.stop = [&] { ++retry_stop_count; }; + CHECK_TRUE(hub.registerSource(descriptor, std::move(failed_callbacks), 2)); + auto failed = hub.subscribe(descriptor->id); + CHECK_TRUE(!failed.valid()); + auto retry = hub.subscribe(descriptor->id); + CHECK_TRUE(retry.valid()); + retry.reset(); + CHECK_TRUE(start_attempts == 2); + CHECK_TRUE(retry_stop_count == 1); + CHECK_TRUE(hub.unregisterSource(descriptor->id)); + + std::atomic stop_count{0}; + MediaSourceHub::SourceCallbacks callbacks; + callbacks.start = [](const MediaSourceHub::FrameSink&, + const MediaSourceHub::CancelPredicate&) { + return true; + }; + callbacks.stop = [&] { ++stop_count; }; + CHECK_TRUE(hub.registerSource(descriptor, std::move(callbacks), 2)); + auto live = hub.subscribe(descriptor->id); + CHECK_TRUE(live.valid()); + hub.shutdown(); + CHECK_TRUE(stop_count == 1); + CHECK_TRUE(!live.valid()); + CHECK_TRUE(!live.waitRead(50ms).has_value()); +} + +void testKeyFrameRequestIsOrderedBeforeStop() { + MediaSourceHub hub; + const auto descriptor = makeVideoDescriptor(Codec::H264, 1); + std::atomic key_frame_entered{false}; + std::atomic release_key_frame{false}; + std::atomic stop_count{0}; + + MediaSourceHub::SourceCallbacks callbacks; + callbacks.start = [](const MediaSourceHub::FrameSink&, + const MediaSourceHub::CancelPredicate&) { + return true; + }; + callbacks.stop = [&] { ++stop_count; }; + callbacks.request_key_frame = [&] { + key_frame_entered.store(true, std::memory_order_release); + while (!release_key_frame.load(std::memory_order_acquire)) { + std::this_thread::sleep_for(1ms); + } + return true; + }; + CHECK_TRUE(hub.registerSource(descriptor, std::move(callbacks), 2)); + auto subscription = hub.subscribe(descriptor->id); + CHECK_TRUE(subscription.valid()); + + auto key_frame = std::async(std::launch::async, [&] { + return hub.requestKeyFrame(descriptor->id); + }); + const auto enter_deadline = std::chrono::steady_clock::now() + 500ms; + while (!key_frame_entered.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < enter_deadline) { + std::this_thread::sleep_for(1ms); + } + CHECK_TRUE(key_frame_entered.load(std::memory_order_acquire)); + + auto stop = std::async(std::launch::async, [&] { subscription.reset(); }); + CHECK_TRUE(stop.wait_for(20ms) == std::future_status::timeout); + CHECK_TRUE(stop_count.load(std::memory_order_acquire) == 0); + + release_key_frame.store(true, std::memory_order_release); + CHECK_TRUE(key_frame.get()); + CHECK_TRUE(stop.wait_for(500ms) == std::future_status::ready); + stop.get(); + CHECK_TRUE(stop_count.load(std::memory_order_acquire) == 1); + CHECK_TRUE(!hub.requestKeyFrame(descriptor->id)); +} + +void testHubCancelsBlockedStartWithoutBlockingShutdown() { + MediaSourceHub hub; + const auto descriptor = makeVideoDescriptor(Codec::UNKNOWN, 1); + std::atomic start_entered{false}; + std::atomic start_exited{false}; + + MediaSourceHub::SourceCallbacks callbacks; + callbacks.start = [&](const MediaSourceHub::FrameSink&, + const MediaSourceHub::CancelPredicate& cancelled) { + start_entered.store(true, std::memory_order_release); + while (!cancelled()) { + std::this_thread::sleep_for(2ms); + } + start_exited.store(true, std::memory_order_release); + return false; + }; + CHECK_TRUE(hub.registerSource(descriptor, std::move(callbacks), 2)); + + auto subscription_future = std::async(std::launch::async, [&] { + return hub.subscribe(descriptor->id); + }); + const auto start_deadline = std::chrono::steady_clock::now() + 500ms; + while (!start_entered.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < start_deadline) { + std::this_thread::sleep_for(2ms); + } + + auto shutdown_future = std::async(std::launch::async, [&] { hub.shutdown(); }); + const bool shutdown_completed = shutdown_future.wait_for(500ms) == + std::future_status::ready; + if (shutdown_completed) shutdown_future.get(); + const bool subscribe_completed = subscription_future.wait_for(500ms) == + std::future_status::ready; + bool invalid_subscription = false; + if (subscribe_completed) { + invalid_subscription = !subscription_future.get().valid(); + } + const auto exit_deadline = std::chrono::steady_clock::now() + 500ms; + while (!start_exited.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < exit_deadline) { + std::this_thread::sleep_for(2ms); + } + + CHECK_TRUE(start_entered.load(std::memory_order_acquire)); + CHECK_TRUE(shutdown_completed); + CHECK_TRUE(subscribe_completed); + CHECK_TRUE(invalid_subscription); + CHECK_TRUE(start_exited.load(std::memory_order_acquire)); +} + +void testHubQuarantinesNonCooperativeStart() { + MediaSourceHub hub; + const auto descriptor = makeVideoDescriptor(Codec::UNKNOWN, 1); + std::atomic start_entered{false}; + std::atomic release_start{false}; + std::atomic start_exited{false}; + std::atomic stop_count{0}; + + MediaSourceHub::SourceCallbacks callbacks; + callbacks.start = [&](const MediaSourceHub::FrameSink&, + const MediaSourceHub::CancelPredicate&) { + start_entered.store(true, std::memory_order_release); + while (!release_start.load(std::memory_order_acquire)) { + std::this_thread::sleep_for(2ms); + } + start_exited.store(true, std::memory_order_release); + return true; + }; + callbacks.stop = [&] { ++stop_count; }; + CHECK_TRUE(hub.registerSource(descriptor, std::move(callbacks), 2)); + + auto subscription_future = std::async(std::launch::async, [&] { + return hub.subscribe(descriptor->id); + }); + const auto start_deadline = std::chrono::steady_clock::now() + 500ms; + while (!start_entered.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < start_deadline) { + std::this_thread::sleep_for(2ms); + } + + const auto shutdown_started = std::chrono::steady_clock::now(); + hub.shutdown(); + const bool shutdown_was_bounded = + std::chrono::steady_clock::now() - shutdown_started < 500ms; + const bool subscribe_completed = subscription_future.wait_for(500ms) == + std::future_status::ready; + bool invalid_subscription = false; + if (subscribe_completed) { + invalid_subscription = !subscription_future.get().valid(); + } + + // Release the deliberately non-cooperative test callback before its stack + // captures go out of scope. Late successful startup must be stopped once. + release_start.store(true, std::memory_order_release); + const auto exit_deadline = std::chrono::steady_clock::now() + 500ms; + while ((!start_exited.load(std::memory_order_acquire) || stop_count.load() != 1) && + std::chrono::steady_clock::now() < exit_deadline) { + std::this_thread::sleep_for(2ms); + } + + CHECK_TRUE(start_entered.load(std::memory_order_acquire)); + CHECK_TRUE(shutdown_was_bounded); + CHECK_TRUE(subscribe_completed); + CHECK_TRUE(invalid_subscription); + CHECK_TRUE(start_exited.load(std::memory_order_acquire)); + CHECK_TRUE(stop_count.load() == 1); +} + +} // namespace + +int main() { + testMediaMetadataValidation(); + testLegacySpmcCompatibility(); + testBroadcastFrameRing(); + testBroadcastConcurrency(); + testHubLifecycleAndDescriptorRefresh(); + testHubFailedStartAndShutdown(); + testKeyFrameRequestIsOrderedBeforeStop(); + testHubCancelsBlockedStartWithoutBlockingShutdown(); + testHubQuarantinesNonCooperativeStart(); + + if (failures != 0) { + std::cerr << failures << " media_source_hub checks failed\n"; + return 1; + } + std::cout << "media_source_hub self-test passed\n"; + return 0; +} diff --git a/cmvr-es/manager/task_manager/src/task_manager.cpp b/cmvr-es/manager/task_manager/src/task_manager.cpp index 05410231..7b7f3f9d 100644 --- a/cmvr-es/manager/task_manager/src/task_manager.cpp +++ b/cmvr-es/manager/task_manager/src/task_manager.cpp @@ -36,6 +36,8 @@ const char* taskConfigTypeToString(const config::TaskConfigEntry::TaskType type) return "TASK_TYPE_TOUCH_SCREEN"; case config::TaskConfigEntry::TASK_TYPE_GRPC_SERVER: return "TASK_TYPE_GRPC_SERVER"; + case config::TaskConfigEntry::TASK_TYPE_QUIC_EDGE: + return "TASK_TYPE_QUIC_EDGE"; case config::TaskConfigEntry::TASK_TYPE_UNKNOWN: default: return "TASK_TYPE_UNKNOWN"; diff --git a/cmvr-es/service/CMakeLists.txt b/cmvr-es/service/CMakeLists.txt index 70f9cd95..940f7f3d 100644 --- a/cmvr-es/service/CMakeLists.txt +++ b/cmvr-es/service/CMakeLists.txt @@ -21,6 +21,8 @@ target_link_libraries(service PRIVATE cmvr_es::task_manager cmvr_es::algorithms::controller cmvr_es::task + cmvr_es::media_source_hub + cmvr_es::device_media_source_adapter protobuf::libprotobuf ) diff --git a/cmvr-es/service/README.md b/cmvr-es/service/README.md new file mode 100644 index 00000000..6802b950 --- /dev/null +++ b/cmvr-es/service/README.md @@ -0,0 +1,205 @@ +# Service 模块开发指南 + +`service/` 实现边缘端对外协议和设备抽象之间的适配。Service 负责解析请求、查找设备、转换 DTO 和返回结果，不负责创建具体设备后端。 + +返回[项目总览](../../README.md)。 + +## 当前结构 + +| 目录 | 职责 | +| --- | --- | +| `grpc/` | 入站设备控制、状态查询和兼容流式接口 | +| `quic_edge/` | 边缘端主动连接平台的 QUIC client、控制状态机和媒体 packetizer | +| `quic_edge/tests/` | 已登记到 CTest 的 QUIC 协议测试 | + +两个遗留 gRPC client test 位于 `grpc/src/*_client_test.cpp`，当前没有通过 `add_test()` 登记。 + +gRPC 和 QUIC 的职责边界： + +- 机械臂、AGV 等可靠控制继续使用 gRPC； +- 节点注册、心跳和 IP 上报使用 QUIC reliable stream； +- 实时音视频使用 QUIC DATAGRAM； +- `quic_edge/` 不是平台 Gateway，也不是浏览器服务器。 + +## 新增 gRPC Service + +当前没有动态 service registry，必须完成以下全部步骤。 + +### 1. 定义 Proto + +在 [`../../protos/cmvr/api/`](../../protos/cmvr/api/) 增加或扩展： + +- `_command.proto` +- `_service.proto` + +import 路径必须相对于 `protos/`。兼容规则见 [`../../protos/README.md`](../../protos/README.md)。 + +### 2. 实现 Service + +目录约定： + +```text +service/grpc/ +├── include/grpc_example_service.h +└── src/grpc_example_service.cpp +``` + +实现类继承生成的： + +```cpp +cmvr::api::ExampleService::Service +``` + +通过已初始化的 `DeviceManager` 获取抽象设备。不要在 service 中创建厂商 SDK 对象，不要绕过设备 factory。 + +### 3. 加入 service target + +将实现 `.cpp` 加入 [`CMakeLists.txt`](CMakeLists.txt) 的 `service` library，并声明最小依赖。 + +### 4. 注册到 GrpcServerTask + +还必须修改： + +- [`../task/grpc_server_task/include/grpc_server_task.h`](../task/grpc_server_task/include/grpc_server_task.h) +- [`../task/grpc_server_task/src/grpc_server_task.cpp`](../task/grpc_server_task/src/grpc_server_task.cpp) + +完成： + +1. 增加 service owner； +2. 在 start 中构造； +3. 调用 `builder.RegisterService(...)`； +4. 在 `clearServices()` 中 reset。 + +漏掉该步骤时项目可能编译成功，但服务不会出现在 reflection 或运行时。 + +### 5. 测试 + +- 直接测试 service handler 或启动临时 gRPC server； +- 覆盖设备不存在、类型不匹配、设备错误和取消； +- 使用 grpcurl/reflection 验证服务全名； +- 流式 RPC 覆盖客户端断开和慢消费者； +- 在 CMake 中使用 `if(BUILD_TESTING)` 包裹测试目标，并通过 `add_test()` 登记。 + +`grpc_arm_client_test` 和 `grpc_hlc_client_test` 是未登记到 CTest 的历史可执行文件，不能代表默认自动覆盖。 + +## gRPC 实现约束 + +### 错误语义 + +当前历史服务存在两种风格： + +- gRPC status 返回 OK，业务失败写入 Feedback header； +- 使用非 OK gRPC status 表达 transport/API 失败。 + +扩展已有服务时保持其兼容语义。新增服务必须在设计时明确： + +- 哪些错误使用 gRPC status； +- 哪些错误使用业务 Feedback； +- 是否允许部分成功； +- deadline/cancellation 如何映射； +- 不得同时返回互相矛盾的 transport 和业务状态。 + +### 流式 RPC + +- 检查 `context->IsCancelled()`； +- 检查 `Read()` / `Write()` 返回； +- 使用 RAII 或 MediaSourceHub Subscription 释放 producer lease； +- 不持有设备状态锁进行网络写； +- 为 wait/read 使用有限 timeout； +- 慢客户端不能阻塞设备生产线程； +- H.264/H.265 丢帧后等待关键帧恢复。 + +当前仅 gRPC RGB 和麦克风流使用 MediaSourceHub；Depth/RGBD 仍直接读取设备帧。 + +### 当前安全状态 + +GrpcServerTask 使用同步 `grpc::ServerBuilder` 和 `grpc::InsecureServerCredentials()`。reflection 由配置控制。当前没有 gRPC TLS、认证、授权或标准 health service。 + +QUIC 配置中的 `grpc_endpoint_tls` 只是上报字段，不会启用 gRPC TLS。 + +## 扩展 QUIC Edge + +关键层次： + +| 层 | 主要文件 | +| --- | --- | +| 控制流 framing | `quic_edge/src/control_framing.cpp` | +| DATAGRAM 固定头和分片 | `quic_edge/src/datagram_packetizer.cpp` | +| 会话和状态机 | `quic_edge/src/quic_edge_service.cpp` | +| 传输抽象 | `quic_edge/include/quic_transport.h` | +| MsQuic 后端 | `quic_edge/src/msquic_transport.cpp` | +| 设备媒体适配 | `quic_edge/src/quic_edge_device_adapter.cpp` | + +线协议见 [`../../protos/cmvr/quic_edge/v1/README.md`](../../protos/cmvr/quic_edge/v1/README.md)。 + +### 新增 Transport 后端 + +1. 实现 `QuicTransport` 完整接口； +2. 明确 callback 所在线程； +3. stop/close 后不得再访问已销毁 service； +4. `QUEUED` 表示 transport 接管待发送数据； +5. `WOULD_BLOCK` 或 `ERROR` 不得接管任何字节； +6. DATAGRAM batch 本地准入必须原子； +7. native send 部分失败时关闭连接并清理 session； +8. 添加 fake transport 故障注入测试； +9. 在默认 transport factory 中显式选择后端。 + +### 新增控制消息 + +不能只修改 Proto，还要同步： + +- envelope 构造与发送； +- 入站 dispatch； +- 合法状态和消息时序； +- message sequence 校验； +- session ID 和 heartbeat sequence 校验； +- reconnect 后状态清理； +- Java Gateway 对端； +- framing、状态机和 fake transport 测试。 + +当前 Edge 入站只接受： + +- `NodeRegisterResponse` +- `NodeHeartbeatAck` +- `ProtocolError` + +虽然 Proto 定义了 `MediaSessionClose`，本版本 Edge 收到它仍会判为 unexpected，不应将其描述为已实现的双向控制能力。 + +### 跨 QUIC 通道顺序 + +Edge 会先调用可靠流发送 session/descriptor，再调用 DATAGRAM 发送媒体，但 QUIC stream 与 DATAGRAM 没有跨通道到达顺序保证。 + +Gateway 必须容忍 DATAGRAM 先到，对未知 session epoch 或 codec generation 的数据有界暂存或丢弃。 + +## QUIC 测试要求 + +参考 [`quic_edge/CMakeLists.txt`](quic_edge/CMakeLists.txt) 和 `quic_edge_protocol_test`，至少覆盖： + +- 控制消息拆包、粘包和超限； +- 重复或倒退 sequence； +- 注册、ACK 超时和重连； +- session epoch 清理； +- DATAGRAM header 字节序； +- 分片边界和超大帧； +- 原子队列准入和背压； +- 丢帧、generation 和关键帧恢复； +- stop 与 callback 并发。 + +```bash +cmake --build build --target quic_edge_protocol_test +ctest \ + --test-dir build \ + -R '^quic_edge_protocol_test$' \ + --output-on-failure +``` + +## 提交检查 + +- [ ] service 不创建具体硬件后端 +- [ ] Proto、实现、CMake 和 GrpcServerTask 注册均已更新 +- [ ] 错误语义与已有服务兼容 +- [ ] deadline、cancel、Read/Write 失败均处理 +- [ ] 流式资源通过 RAII 释放 +- [ ] gRPC 安全能力没有被配置字段误描述 +- [ ] QUIC 状态机与 Gateway 同步更新 +- [ ] 协议测试已登记到 CTest diff --git a/cmvr-es/service/grpc/src/grpc_camera_service.cpp b/cmvr-es/service/grpc/src/grpc_camera_service.cpp index 7ca3d5a4..15cd5871 100644 --- a/cmvr-es/service/grpc/src/grpc_camera_service.cpp +++ b/cmvr-es/service/grpc/src/grpc_camera_service.cpp @@ -1,10 +1,14 @@ #include "common/base/logging/logger.h" +#include "manager/media_source_hub/include/device_media_source_adapter.h" // // Created by xtkuang on 2025/6/1. // #include "../include/grpc_camera_service.h" +#include +#include +#include #include using namespace std; @@ -61,6 +65,41 @@ bool toPtzCommand(cmvr::api::ControlPtzCommand_Command command, PtzCommand& out) return false; } } + +// The legacy depth/RGBD RPCs acquire the camera's shared producer directly +// instead of going through MediaSourceHub. Keep that lease exception-safe: +// cancellation, a failed Write(), or any conversion error must release exactly +// the one startStreaming() reference acquired by this call. +class CameraStreamingLease final { +public: + explicit CameraStreamingLease(std::shared_ptr camera) + : camera_(std::move(camera)) { + active_ = camera_ && camera_->startStreaming(); + } + + ~CameraStreamingLease() { + if (!active_ || !camera_) { + return; + } + try { + camera_->stopStreaming(); + } catch (const std::exception& e) { + CMVR_LOG(ERROR) << "[gRPCCameraServiceImpl] failed to release camera stream lease: " + << e.what(); + } catch (...) { + CMVR_LOG(ERROR) << "[gRPCCameraServiceImpl] failed to release camera stream lease"; + } + } + + CameraStreamingLease(const CameraStreamingLease&) = delete; + CameraStreamingLease& operator=(const CameraStreamingLease&) = delete; + + explicit operator bool() const noexcept { return active_; } + +private: + std::shared_ptr camera_; + bool active_{false}; +}; } gRPCCameraServiceImpl::gRPCCameraServiceImpl(): dmgr_(DeviceManager::getInstance()) {} @@ -391,6 +430,10 @@ grpc::Status gRPCCameraServiceImpl::ControlPtz(grpc::ServerContext* context, return failResponse(response, "Invalid PTZ command"); } + if (request->action() != api::ControlPtzCommand_Action_START && + request->action() != api::ControlPtzCommand_Action_STOP) { + return failResponse(response, "Invalid PTZ action"); + } const bool stop = request->action() == api::ControlPtzCommand_Action_STOP; if (!dev->controlPtz(command, stop, static_cast(request->speed()))) { CameraState state{}; @@ -418,9 +461,11 @@ grpc::Status gRPCCameraServiceImpl::GetDepthImageStream(grpc::ServerContext* con try { //读取首次传递的数据，获取设备id api::GetDepthImageStreamCommand_Request request; - stream->Read(&request); + if (!stream->Read(&request)) { + return grpc::Status::OK; + } string dev_id = request.header().device_id(); - CMVR_LOG(INFO) << "[gRPCCameraServiceImpl] (GetRGBImageStream): start,id=" << dev_id; + CMVR_LOG(INFO) << "[gRPCCameraServiceImpl] (GetDepthImageStream): start,id=" << dev_id; const auto dev = dmgr_.getDevice(dev_id); if (!dev) { api::GetDepthImageStreamCommand_Feedback response; @@ -430,8 +475,17 @@ grpc::Status gRPCCameraServiceImpl::GetDepthImageStream(grpc::ServerContext* con stream->Write(response); return grpc::Status::OK; } + CameraStreamingLease stream_lease(dev); + if (!stream_lease) { + api::GetDepthImageStreamCommand_Feedback response; + response.mutable_header()->set_success(false); + response.mutable_header()->set_error_message("Failed to start camera stream: " + dev_id); + setCurrentTimestamp(response.mutable_header()->mutable_timestamp()); + stream->Write(response); + return grpc::Status::OK; + } + int nFrameCount = 0; - dev->startStreaming(); size_t index = 0; while (true) { @@ -443,9 +497,10 @@ grpc::Status gRPCCameraServiceImpl::GetDepthImageStream(grpc::ServerContext* con api::GetDepthImageStreamCommand_Feedback response; cmvr::device::StreamFrameData frame_data; - dev->getEncodedFrame(frame_data,index); - if (!frame_data.rgbFrame.empty()) { + if (dev->waitEncodedFrame(frame_data, index, std::chrono::milliseconds(100)) && + !frame_data.depthFrame.empty()) { response.mutable_header()->set_success(true); + setCurrentTimestamp(response.mutable_header()->mutable_timestamp()); response.mutable_depth_frame()->set_data(frame_data.depthFrame.data(), frame_data.depthFrame.size()); response.mutable_depth_frame()->set_is_key_frame(frame_data.depthKey); response.mutable_depth_frame()->set_codec(frame_data.codec); @@ -453,29 +508,27 @@ grpc::Status gRPCCameraServiceImpl::GetDepthImageStream(grpc::ServerContext* con response.mutable_depth_frame()->set_height(frame_data.height); response.mutable_intrinsics()->set_fx(frame_data.intrinsics.fx); - response.mutable_intrinsics()->set_fy(frame_data.intrinsics.fx); - response.mutable_intrinsics()->set_cx(frame_data.intrinsics.fx); - response.mutable_intrinsics()->set_cy(frame_data.intrinsics.fx); + response.mutable_intrinsics()->set_fy(frame_data.intrinsics.fy); + response.mutable_intrinsics()->set_cx(frame_data.intrinsics.cx); + response.mutable_intrinsics()->set_cy(frame_data.intrinsics.cy); for (int i = 0; i < 5 ; i++) { response.mutable_intrinsics()->add_coeffs(frame_data.intrinsics.coeffs[i]); } response.set_seq_no(nFrameCount++); - grpc::WriteOptions options; - options.set_last_message(); if (!stream->Write(response)) { CMVR_LOG(INFO) << "[gRPCCameraServiceImpl] (stream->Write) failed,id=" << dev_id; break; } } } - CMVR_LOG(INFO) << "[gRPCCameraServiceImpl] (GetRGBImageStream): end,id=" << dev_id; - dev->stopStreaming(); + CMVR_LOG(INFO) << "[gRPCCameraServiceImpl] (GetDepthImageStream): end,id=" << dev_id; return grpc::Status::OK; } - catch (exception &e) { + catch (const exception &e) { api::GetDepthImageStreamCommand_Feedback response; + response.mutable_header()->set_success(false); response.mutable_header()->set_error_message(e.what()); setCurrentTimestamp(response.mutable_header()->mutable_timestamp()); stream->Write(response); @@ -487,9 +540,11 @@ grpc::Status gRPCCameraServiceImpl::GetRGBDImagesStream(grpc::ServerContext* con try { //读取首次传递的数据，获取设备id api::GetRGBDImagesStreamCommand_Request request; - stream->Read(&request); + if (!stream->Read(&request)) { + return grpc::Status::OK; + } string dev_id = request.header().device_id(); - CMVR_LOG(INFO) << "[gRPCCameraServiceImpl] (GetRGBImageStream): start,id=" << dev_id; + CMVR_LOG(INFO) << "[gRPCCameraServiceImpl] (GetRGBDImagesStream): start,id=" << dev_id; const auto dev = dmgr_.getDevice(dev_id); if (!dev) { api::GetRGBDImagesStreamCommand_Feedback response; @@ -499,8 +554,17 @@ grpc::Status gRPCCameraServiceImpl::GetRGBDImagesStream(grpc::ServerContext* con stream->Write(response); return grpc::Status::OK; } + CameraStreamingLease stream_lease(dev); + if (!stream_lease) { + api::GetRGBDImagesStreamCommand_Feedback response; + response.mutable_header()->set_success(false); + response.mutable_header()->set_error_message("Failed to start camera stream: " + dev_id); + setCurrentTimestamp(response.mutable_header()->mutable_timestamp()); + stream->Write(response); + return grpc::Status::OK; + } + int nFrameCount = 0; - dev->startStreaming(); size_t index = 0; while (true) { @@ -512,9 +576,11 @@ grpc::Status gRPCCameraServiceImpl::GetRGBDImagesStream(grpc::ServerContext* con api::GetRGBDImagesStreamCommand_Feedback response; cmvr::device::StreamFrameData frame_data; - dev->getEncodedFrame(frame_data,index); - if (!frame_data.rgbFrame.empty()) { + if (dev->waitEncodedFrame(frame_data, index, std::chrono::milliseconds(100)) && + !frame_data.rgbFrame.empty() && + !frame_data.depthFrame.empty()) { response.mutable_header()->set_success(true); + setCurrentTimestamp(response.mutable_header()->mutable_timestamp()); response.mutable_color_frame()->set_data(frame_data.rgbFrame.data(), frame_data.rgbFrame.size()); response.mutable_color_frame()->set_is_key_frame(frame_data.bKey); response.mutable_color_frame()->set_codec(frame_data.codec); @@ -528,17 +594,15 @@ grpc::Status gRPCCameraServiceImpl::GetRGBDImagesStream(grpc::ServerContext* con response.mutable_depth_frame()->set_height(frame_data.height); response.mutable_intrinsics()->set_fx(frame_data.intrinsics.fx); - response.mutable_intrinsics()->set_fy(frame_data.intrinsics.fx); - response.mutable_intrinsics()->set_cx(frame_data.intrinsics.fx); - response.mutable_intrinsics()->set_cy(frame_data.intrinsics.fx); + response.mutable_intrinsics()->set_fy(frame_data.intrinsics.fy); + response.mutable_intrinsics()->set_cx(frame_data.intrinsics.cx); + response.mutable_intrinsics()->set_cy(frame_data.intrinsics.cy); for (int i = 0; i < 5 ; i++) { response.mutable_intrinsics()->add_coeffs(frame_data.intrinsics.coeffs[i]); } response.set_seq_no(nFrameCount++); - grpc::WriteOptions options; - options.set_last_message(); if (!stream->Write(response)) { CMVR_LOG(INFO) << "[gRPCCameraServiceImpl] (stream->Write) failed,id=" << dev_id; break; @@ -546,11 +610,11 @@ grpc::Status gRPCCameraServiceImpl::GetRGBDImagesStream(grpc::ServerContext* con } } CMVR_LOG(INFO) << "[gRPCCameraServiceImpl] (GetRGBDImagesStream): end,id=" << dev_id; - dev->stopStreaming(); return grpc::Status::OK; } - catch (exception &e) { + catch (const exception &e) { api::GetRGBDImagesStreamCommand_Feedback response; + response.mutable_header()->set_success(false); response.mutable_header()->set_error_message(e.what()); setCurrentTimestamp(response.mutable_header()->mutable_timestamp()); stream->Write(response); @@ -561,7 +625,9 @@ grpc::Status gRPCCameraServiceImpl::GetRGBImageStream(grpc::ServerContext* conte try { //读取首次传递的数据，获取设备id api::GetRGBImageStreamCommand_Request request; - stream->Read(&request); + if (!stream->Read(&request)) { + return grpc::Status::OK; + } string dev_id = request.header().device_id(); CMVR_LOG(INFO) << "[gRPCCameraServiceImpl] (GetRGBImageStream): start,id=" << dev_id; const auto dev = dmgr_.getDevice(dev_id); @@ -573,9 +639,39 @@ grpc::Status gRPCCameraServiceImpl::GetRGBImageStream(grpc::ServerContext* conte stream->Write(response); return grpc::Status::OK; } - int nFrameCount = 0; - dev->startStreaming(); - size_t last_sent_index = std::numeric_limits::max(); + auto& media_hub = cmvr::media::globalMediaSourceHub(); + const std::string track_id = cmvr::media::cameraColorTrackId(dev_id); + if (!cmvr::media::ensureCameraMediaSource(media_hub, dev)) { + api::GetRGBImageStreamCommand_Feedback response; + response.mutable_header()->set_success(false); + response.mutable_header()->set_error_message("Failed to register camera media source: " + dev_id); + setCurrentTimestamp(response.mutable_header()->mutable_timestamp()); + stream->Write(response); + return grpc::Status::OK; + } + auto subscription = media_hub.subscribe( + track_id, + cmvr::media::MediaSourceHub::StartPosition::NEXT_PUBLISHED, + [context] { return context->IsCancelled(); }); + if (!subscription) { + api::GetRGBImageStreamCommand_Feedback response; + response.mutable_header()->set_success(false); + response.mutable_header()->set_error_message("Failed to subscribe camera media source: " + dev_id); + setCurrentTimestamp(response.mutable_header()->mutable_timestamp()); + stream->Write(response); + return grpc::Status::OK; + } + + bool waiting_for_key_frame = true; + auto last_key_frame_request = std::chrono::steady_clock::now(); + media_hub.requestKeyFrame(track_id); + const auto request_key_frame_if_due = [&] { + const auto now = std::chrono::steady_clock::now(); + if (now - last_key_frame_request >= std::chrono::milliseconds(500)) { + media_hub.requestKeyFrame(track_id); + last_key_frame_request = now; + } + }; while (true) { if (context->IsCancelled()) @@ -584,45 +680,77 @@ grpc::Status gRPCCameraServiceImpl::GetRGBImageStream(grpc::ServerContext* conte break; } - api::GetRGBImageStreamCommand_Feedback response; - cmvr::device::StreamFrameData frame_data; - size_t next_index = last_sent_index; - if (!dev->getLatestEncodedFrame(frame_data, next_index) || - frame_data.rgbFrame.empty() || - next_index == last_sent_index) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + const auto read = subscription.waitRead(std::chrono::milliseconds(100)); + if (!read || !read->value || read->value->empty()) { + if (!subscription.valid()) { + break; + } + if (waiting_for_key_frame) { + request_key_frame_if_due(); + } continue; } + const auto& frame = *read->value; + const auto descriptor = frame.descriptor; + if (!descriptor) { + continue; + } + const bool inter_frame_codec = descriptor->codec == cmvr::media::Codec::H264 || + descriptor->codec == cmvr::media::Codec::H265; + if (!inter_frame_codec || descriptor->payload_format != cmvr::media::PayloadFormat::ANNEX_B) { + api::GetRGBImageStreamCommand_Feedback response; + response.mutable_header()->set_success(false); + response.mutable_header()->set_error_message( + "Unsupported camera stream codec or payload format: " + dev_id); + setCurrentTimestamp(response.mutable_header()->mutable_timestamp()); + stream->Write(response); + break; + } + if (read->dropped_since_last_read > 0 || read->generation_changed || frame.discontinuity) { + waiting_for_key_frame = true; + media_hub.requestKeyFrame(track_id); + last_key_frame_request = std::chrono::steady_clock::now(); + } + if (waiting_for_key_frame && !frame.key_frame) { + request_key_frame_if_due(); + continue; + } + waiting_for_key_frame = false; + api::GetRGBImageStreamCommand_Feedback response; response.mutable_header()->set_success(true); - response.mutable_color_frame()->set_data(frame_data.rgbFrame.data(), frame_data.rgbFrame.size()); - response.mutable_color_frame()->set_is_key_frame(frame_data.bKey); - response.mutable_color_frame()->set_codec(frame_data.codec); - response.mutable_color_frame()->set_width(frame_data.width); - response.mutable_color_frame()->set_height(frame_data.height); + setCurrentTimestamp(response.mutable_header()->mutable_timestamp()); + response.mutable_color_frame()->set_data(frame.data(), frame.size()); + response.mutable_color_frame()->set_is_key_frame(frame.key_frame); + response.mutable_color_frame()->set_codec( + descriptor->codec == cmvr::media::Codec::H264 ? "h264" : + descriptor->codec == cmvr::media::Codec::H265 ? "h265" : "unknown"); + response.mutable_color_frame()->set_width(static_cast(descriptor->width)); + response.mutable_color_frame()->set_height(static_cast(descriptor->height)); - response.mutable_intrinsics()->set_fx(frame_data.intrinsics.fx); - response.mutable_intrinsics()->set_fy(frame_data.intrinsics.fy); - response.mutable_intrinsics()->set_cx(frame_data.intrinsics.cx); - response.mutable_intrinsics()->set_cy(frame_data.intrinsics.cy); - for (int i = 0; i < 5 ; i++) { - response.mutable_intrinsics()->add_coeffs(frame_data.intrinsics.coeffs[i]); + response.mutable_intrinsics()->set_fx(descriptor->fx); + response.mutable_intrinsics()->set_fy(descriptor->fy); + response.mutable_intrinsics()->set_cx(descriptor->cx); + response.mutable_intrinsics()->set_cy(descriptor->cy); + for (const float coefficient : descriptor->distortion) { + response.mutable_intrinsics()->add_coeffs(coefficient); } - response.set_seq_no(nFrameCount++); + response.set_seq_no(static_cast(std::min( + frame.sequence, + static_cast(std::numeric_limits::max())))); if (!stream->Write(response)) { CMVR_LOG(INFO) << "[gRPCCameraServiceImpl] (stream->Write) failed,id=" << dev_id; break; } - last_sent_index = next_index; } CMVR_LOG(INFO) << "[gRPCCameraServiceImpl] (GetRGBImageStream): end,id=" << dev_id; - dev->stopStreaming(); return grpc::Status::OK; } - catch (exception &e) { + catch (const exception &e) { api::GetRGBImageStreamCommand_Feedback response; + response.mutable_header()->set_success(false); response.mutable_header()->set_error_message(e.what()); setCurrentTimestamp(response.mutable_header()->mutable_timestamp()); stream->Write(response); diff --git a/cmvr-es/service/grpc/src/grpc_microphone_service.cpp b/cmvr-es/service/grpc/src/grpc_microphone_service.cpp index c4e20e7c..510a7b64 100644 --- a/cmvr-es/service/grpc/src/grpc_microphone_service.cpp +++ b/cmvr-es/service/grpc/src/grpc_microphone_service.cpp @@ -1,6 +1,9 @@ #include "common/base/logging/logger.h" +#include "manager/media_source_hub/include/device_media_source_adapter.h" +#include #include -#include +#include +#include // // Created by linbo on 2025/6/13. // Created by xtkuang on 2025/6/13. @@ -21,30 +24,6 @@ grpc::Status failResponse(ResponseT* response, const std::string& message) { return grpc::Status::OK; } -cmvr::api::AudioData_AudioFormat toProtoAudioFormat(AudioStreamFormat format) { - switch (format) { - case AudioStreamFormat::PCM: - return cmvr::api::AudioData_AudioFormat_PCM; - case AudioStreamFormat::MP3: - return cmvr::api::AudioData_AudioFormat_MP3; - case AudioStreamFormat::AAC: - return cmvr::api::AudioData_AudioFormat_AAC; - case AudioStreamFormat::WAV: - return cmvr::api::AudioData_AudioFormat_WAV; - default: - return cmvr::api::AudioData_AudioFormat_PCM; - } -} - -void fillProtoAudioData(const AudioStreamFrameData& frame, cmvr::api::AudioData* audio) { - audio->set_data(frame.data.data(), frame.data.size()); - audio->set_sample_rate(frame.sample_rate); - audio->set_channels(frame.channels); - audio->set_format(toProtoAudioFormat(frame.format)); - audio->set_codec(frame.codec); - audio->set_pts(frame.pts); - audio->set_nb_samples(frame.nb_samples); -} } gRPCMicroPhoneServiceImpl::gRPCMicroPhoneServiceImpl(): dmgr_(DeviceManager::getInstance()) {} @@ -171,56 +150,106 @@ grpc::Status gRPCMicroPhoneServiceImpl::ResumeRecord(grpc::ServerContext* contex grpc::Status gRPCMicroPhoneServiceImpl::StreamAudio(grpc::ServerContext* context, const api::StreamMicAudioCommand_Request* request, grpc::ServerWriter* writer) { - const string dev_id = request->header().device_id(); - CMVR_LOG(INFO) << "[gRPCMicroPhoneServiceImpl] (StreamAudio): id=" << dev_id; - const auto dev = dmgr_.getDevice(dev_id); - if (!dev) { - api::StreamMicAudioCommand_Feedback feedback; - feedback.mutable_header()->set_success(false); - feedback.mutable_header()->set_error_message("Microphone device not found: " + dev_id); - setCurrentTimestamp(feedback.mutable_header()->mutable_timestamp()); - writer->Write(feedback); - return grpc::Status::OK; - } - - if (!dev->start()) { - api::StreamMicAudioCommand_Feedback feedback; - feedback.mutable_header()->set_success(false); - feedback.mutable_header()->set_error_message("Failed to start microphone: " + dev_id); - setCurrentTimestamp(feedback.mutable_header()->mutable_timestamp()); - writer->Write(feedback); - return grpc::Status::OK; - } - - if (!dev->startStreaming()) { - api::StreamMicAudioCommand_Feedback feedback; - feedback.mutable_header()->set_success(false); - feedback.mutable_header()->set_error_message("Failed to start microphone streaming: " + dev_id); - setCurrentTimestamp(feedback.mutable_header()->mutable_timestamp()); - writer->Write(feedback); - return grpc::Status::OK; - } - - size_t index = 0; - while (!context->IsCancelled()) { - AudioStreamFrameData frame; - dev->getEncodedFrame(frame, index); - if (frame.data.empty()) { - std::this_thread::sleep_for(std::chrono::milliseconds(2)); - continue; + try { + const string dev_id = request->header().device_id(); + CMVR_LOG(INFO) << "[gRPCMicroPhoneServiceImpl] (StreamAudio): id=" << dev_id; + const auto dev = dmgr_.getDevice(dev_id); + if (!dev) { + api::StreamMicAudioCommand_Feedback feedback; + feedback.mutable_header()->set_success(false); + feedback.mutable_header()->set_error_message("Microphone device not found: " + dev_id); + setCurrentTimestamp(feedback.mutable_header()->mutable_timestamp()); + writer->Write(feedback); + return grpc::Status::OK; } - api::StreamMicAudioCommand_Feedback feedback; - feedback.mutable_header()->set_success(true); - setCurrentTimestamp(feedback.mutable_header()->mutable_timestamp()); - fillProtoAudioData(frame, feedback.mutable_audio()); - if (!writer->Write(feedback)) { - break; + auto& media_hub = cmvr::media::globalMediaSourceHub(); + const std::string track_id = cmvr::media::microphoneTrackId(dev_id); + if (!cmvr::media::ensureMicrophoneMediaSource(media_hub, dev)) { + api::StreamMicAudioCommand_Feedback feedback; + feedback.mutable_header()->set_success(false); + feedback.mutable_header()->set_error_message("Failed to register microphone media source: " + dev_id); + setCurrentTimestamp(feedback.mutable_header()->mutable_timestamp()); + writer->Write(feedback); + return grpc::Status::OK; } - } - dev->stopStreaming(); - return grpc::Status::OK; + auto subscription = media_hub.subscribe( + track_id, + cmvr::media::MediaSourceHub::StartPosition::NEXT_PUBLISHED, + [context] { return context->IsCancelled(); }); + if (!subscription) { + api::StreamMicAudioCommand_Feedback feedback; + feedback.mutable_header()->set_success(false); + feedback.mutable_header()->set_error_message("Failed to subscribe microphone media source: " + dev_id); + setCurrentTimestamp(feedback.mutable_header()->mutable_timestamp()); + writer->Write(feedback); + return grpc::Status::OK; + } + + while (!context->IsCancelled()) { + const auto read = subscription.waitRead(std::chrono::milliseconds(100)); + if (!read || !read->value || read->value->empty()) { + if (!subscription.valid()) { + break; + } + continue; + } + const auto& frame = *read->value; + const auto descriptor = frame.descriptor; + if (!descriptor) { + continue; + } + + api::StreamMicAudioCommand_Feedback feedback; + feedback.mutable_header()->set_success(true); + setCurrentTimestamp(feedback.mutable_header()->mutable_timestamp()); + auto* audio = feedback.mutable_audio(); + audio->set_data(frame.data(), frame.size()); + audio->set_sample_rate(static_cast(std::min( + descriptor->sample_rate, + static_cast(std::numeric_limits::max())))); + audio->set_channels(static_cast(std::min( + descriptor->channels, + static_cast(std::numeric_limits::max())))); + if (descriptor->codec == cmvr::media::Codec::PCM_S16LE) { + audio->set_format(cmvr::api::AudioData_AudioFormat_PCM); + audio->set_codec("pcm_s16le"); + } else if (descriptor->codec == cmvr::media::Codec::OPUS) { + audio->set_format(cmvr::api::AudioData_AudioFormat_OPUS); + audio->set_codec("opus"); + } else if (descriptor->codec == cmvr::media::Codec::AAC) { + audio->set_format(cmvr::api::AudioData_AudioFormat_AAC); + audio->set_codec("aac"); + } else { + feedback.mutable_header()->set_success(false); + feedback.mutable_header()->set_error_message( + "Unsupported microphone stream codec: " + dev_id); + feedback.clear_audio(); + writer->Write(feedback); + break; + } + audio->set_pts(frame.pts); + const int64_t sample_count = frame.duration > 0 + ? frame.duration + : static_cast(descriptor->nominal_rate); + audio->set_nb_samples(static_cast(std::clamp( + sample_count, + 0, + std::numeric_limits::max()))); + if (!writer->Write(feedback)) { + break; + } + } + return grpc::Status::OK; + } catch (const std::exception& error) { + api::StreamMicAudioCommand_Feedback feedback; + feedback.mutable_header()->set_success(false); + feedback.mutable_header()->set_error_message(error.what()); + setCurrentTimestamp(feedback.mutable_header()->mutable_timestamp()); + writer->Write(feedback); + return grpc::Status::OK; + } } grpc::Status gRPCMicroPhoneServiceImpl::SetVolume(grpc::ServerContext* context, diff --git a/cmvr-es/service/quic_edge/CMakeLists.txt b/cmvr-es/service/quic_edge/CMakeLists.txt new file mode 100644 index 00000000..7d104ab6 --- /dev/null +++ b/cmvr-es/service/quic_edge/CMakeLists.txt @@ -0,0 +1,59 @@ +find_package(Threads REQUIRED) + +add_library(quic_edge_service STATIC + src/control_framing.cpp + src/datagram_packetizer.cpp + src/quic_edge_types.cpp + src/quic_transport.cpp + src/quic_edge_service.cpp + src/quic_edge_device_adapter.cpp +) +target_compile_features(quic_edge_service PUBLIC cxx_std_17) +target_include_directories(quic_edge_service PUBLIC ${PROJECT_SOURCE_DIR}/cmvr-es) +target_link_libraries(quic_edge_service + PUBLIC + cmvr_es::proto + cmvr_es::media_source_hub + PRIVATE + cmvr_es::device_media_source_adapter + cmvr_es::device_manager + Threads::Threads +) + +if(CMVR_HAS_MSQUIC) + target_sources(quic_edge_service PRIVATE src/msquic_transport.cpp) + target_compile_definitions(quic_edge_service PRIVATE CMVR_HAS_MSQUIC=1) + target_link_libraries(quic_edge_service PRIVATE MsQuic::msquic) + + # Bundle a repository-local or explicitly supplied MsQuic runtime with the + # existing relocatable output tree. System packages remain system-owned. + string(FIND "${MsQuic_LIBRARY}" "${PROJECT_SOURCE_DIR}/dependency/" + _cmvr_bundled_msquic_index) + if(MsQuic_LIBRARY MATCHES "\\.so" AND + (_cmvr_bundled_msquic_index EQUAL 0 OR CMVR_MSQUIC_ROOT)) + get_filename_component(_cmvr_msquic_library_dir "${MsQuic_LIBRARY}" DIRECTORY) + install(DIRECTORY "${_cmvr_msquic_library_dir}/" DESTINATION lib + FILES_MATCHING PATTERN "libmsquic.so*") + endif() +endif() + +add_library(cmvr_es::quic_edge_service ALIAS quic_edge_service) + +if(BUILD_TESTING) + add_executable(quic_edge_protocol_test tests/quic_edge_protocol_test.cpp) + target_compile_features(quic_edge_protocol_test PRIVATE cxx_std_17) + target_link_libraries(quic_edge_protocol_test + PRIVATE + cmvr_es::quic_edge_service + cmvr_es::media_source_hub + Threads::Threads + ) + add_test(NAME quic_edge_protocol_test COMMAND quic_edge_protocol_test) + if(UNIX AND NOT APPLE) + get_property(_quic_test_library_dirs DIRECTORY PROPERTY LINK_DIRECTORIES) + list(PREPEND _quic_test_library_dirs "${CMAKE_BINARY_DIR}/cmvr_compiler_runtime") + list(JOIN _quic_test_library_dirs ":" _quic_test_library_path) + set_tests_properties(quic_edge_protocol_test PROPERTIES + ENVIRONMENT "LD_LIBRARY_PATH=${_quic_test_library_path}") + endif() +endif() diff --git a/cmvr-es/service/quic_edge/include/control_framing.h b/cmvr-es/service/quic_edge/include/control_framing.h new file mode 100644 index 00000000..def77614 --- /dev/null +++ b/cmvr-es/service/quic_edge/include/control_framing.h @@ -0,0 +1,54 @@ +#ifndef CMVR_ES_QUIC_EDGE_CONTROL_FRAMING_H +#define CMVR_ES_QUIC_EDGE_CONTROL_FRAMING_H + +#include +#include +#include +#include + +namespace cmvr::quic_edge { + +class ControlFrameEncoder { +public: + static bool encode(const std::uint8_t* payload, + std::size_t payload_size, + std::size_t maximum_payload_size, + std::vector* framed, + std::string* error); + + static bool encode(const std::vector& payload, + std::size_t maximum_payload_size, + std::vector* framed, + std::string* error) + { + return encode(payload.data(), payload.size(), maximum_payload_size, framed, error); + } +}; + +class ControlFrameDecoder { +public: + explicit ControlFrameDecoder(std::size_t maximum_payload_size); + + bool push(const std::uint8_t* data, + std::size_t size, + std::vector>* decoded_frames, + std::string* error); + + bool push(const std::vector& data, + std::vector>* decoded_frames, + std::string* error) + { + return push(data.data(), data.size(), decoded_frames, error); + } + + void reset(); + std::size_t bufferedBytes() const { return buffer_.size(); } + +private: + std::size_t maximum_payload_size_; + std::vector buffer_; +}; + +} // namespace cmvr::quic_edge + +#endif // CMVR_ES_QUIC_EDGE_CONTROL_FRAMING_H diff --git a/cmvr-es/service/quic_edge/include/datagram_packetizer.h b/cmvr-es/service/quic_edge/include/datagram_packetizer.h new file mode 100644 index 00000000..6f0d5be2 --- /dev/null +++ b/cmvr-es/service/quic_edge/include/datagram_packetizer.h @@ -0,0 +1,47 @@ +#ifndef CMVR_ES_QUIC_EDGE_DATAGRAM_PACKETIZER_H +#define CMVR_ES_QUIC_EDGE_DATAGRAM_PACKETIZER_H + +#include +#include +#include +#include + +#include "service/quic_edge/include/quic_edge_types.h" + +namespace cmvr::quic_edge { + +class DatagramPacketizer { +public: + explicit DatagramPacketizer(std::uint64_t initial_packet_sequence = 0); + + bool packetize(const media::MediaFrame& frame, + std::uint32_t wire_track_id, + std::uint32_t codec_generation_token, + std::uint64_t session_epoch, + std::uint64_t wire_frame_sequence, + bool force_discontinuity, + std::size_t maximum_datagram_bytes, + std::vector* packets, + std::string* error); + + static bool decodeHeader(const std::uint8_t* data, + std::size_t size, + DatagramHeader* header, + std::string* error); + + static bool decodeHeader(const std::vector& packet, + DatagramHeader* header, + std::string* error) + { + return decodeHeader(packet.data(), packet.size(), header, error); + } + + std::uint64_t nextPacketSequence() const { return next_packet_sequence_; } + +private: + std::uint64_t next_packet_sequence_; +}; + +} // namespace cmvr::quic_edge + +#endif // CMVR_ES_QUIC_EDGE_DATAGRAM_PACKETIZER_H diff --git a/cmvr-es/service/quic_edge/include/quic_edge_service.h b/cmvr-es/service/quic_edge/include/quic_edge_service.h new file mode 100644 index 00000000..d56055fc --- /dev/null +++ b/cmvr-es/service/quic_edge/include/quic_edge_service.h @@ -0,0 +1,197 @@ +#ifndef CMVR_ES_QUIC_EDGE_SERVICE_H +#define CMVR_ES_QUIC_EDGE_SERVICE_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmvr/config/quic_edge_config/quic_edge_config.pb.h" +#include "manager/media_source_hub/include/media_source_hub.h" +#include "service/quic_edge/include/control_framing.h" +#include "service/quic_edge/include/datagram_packetizer.h" +#include "service/quic_edge/include/quic_transport.h" + +namespace cmvr::quic_edge { + +enum class QuicEdgeServiceState { + UNINITIALIZED, + DISABLED, + STOPPED, + CONNECTING, + REGISTERING, + ONLINE, + BACKOFF, + FAILED, +}; + +const char* toString(QuicEdgeServiceState state); + +struct QuicEdgeStats { + std::uint64_t connection_attempts{0}; + std::uint64_t successful_connections{0}; + std::uint64_t reconnects{0}; + std::uint64_t registrations_sent{0}; + std::uint64_t registrations_accepted{0}; + std::uint64_t registrations_rejected{0}; + std::uint64_t heartbeats_sent{0}; + std::uint64_t heartbeats_acknowledged{0}; + std::uint64_t heartbeat_timeouts{0}; + std::uint64_t protocol_errors{0}; + std::uint64_t media_sessions_opened{0}; + std::uint64_t frames_queued{0}; + std::uint64_t datagrams_queued{0}; + std::uint64_t frames_dropped_source{0}; + std::uint64_t frames_dropped_backpressure{0}; + std::uint64_t frames_dropped_oversize{0}; + std::uint64_t frames_dropped_no_datagram{0}; + std::uint64_t frames_skipped_waiting_keyframe{0}; + std::uint64_t source_errors{0}; +}; + +struct QuicEdgeStatus { + bool registered{false}; + std::string node_id; + std::string boot_id; + std::string session_id; + std::string observed_source_ip; + std::string last_media_error; + std::uint64_t heartbeat_sequence{0}; + std::uint64_t last_heartbeat_ack_unix_ms{0}; + std::size_t active_media_tracks{0}; +}; + +class QuicEdgeService { +public: + explicit QuicEdgeService(config::QuicEdgeConfig config); + QuicEdgeService(config::QuicEdgeConfig config, + std::unique_ptr transport, + media::MediaSourceHub& media_hub); + ~QuicEdgeService(); + + QuicEdgeService(const QuicEdgeService&) = delete; + QuicEdgeService& operator=(const QuicEdgeService&) = delete; + + static bool validateConfig(const config::QuicEdgeConfig& config, + std::string* error); + + bool initialize(std::string* error); + bool start(std::string* error); + void stop(); + + QuicEdgeServiceState state() const; + std::string lastError() const; + QuicEdgeStats stats() const; + QuicEdgeStatus status() const; + bool enabled() const { return config_.enable(); } + +private: + using SourceRegistrar = std::function; + + struct ActiveTrack { + config::QuicEdgeTrackConfig config; + std::string source_track_id; + media::MediaSourceHub::Subscription subscription; + std::optional last_description; + bool waiting_for_keyframe{false}; + bool keyframe_requested{false}; + bool next_frame_discontinuous{false}; + }; + + void run(); + void runMedia(); + bool startMediaWorker(std::string* error); + void stopMediaWorker(); + void recordMediaConnectionFailure(const std::string& error); + bool performRegistration(ControlFrameDecoder* decoder, std::string* error); + bool sendNodeRegistration(std::string* error); + bool sendHeartbeat(std::uint64_t sequence, std::string* error); + bool receiveAndDispatchControl(ControlFrameDecoder* decoder, + std::chrono::milliseconds timeout, + bool* received, + std::string* error); + bool dispatchControlFrame(const std::vector& frame, + std::string* error); + bool openMediaSession(std::vector* tracks, + std::string* error); + void refreshMediaTracks(std::vector* tracks); + bool hasEnabledMediaTracks() const; + bool ensureSourceRegistered(const config::QuicEdgeTrackConfig& track, + const std::string& source_track_id, + std::string* error); + bool sendSessionOpen(std::string* error); + bool sendTrackDescription(const MediaTrackDescription& description, + std::string* error); + bool sendControlEnvelope(const std::string& serialized, + std::string* error); + bool processTrack(ActiveTrack* track, bool* sent_anything, + std::string* error); + std::uint64_t shedBufferedFrames(ActiveTrack* track); + void requestKeyFrame(ActiveTrack* track); + std::uint32_t maximumFrameBytes(const ActiveTrack& track) const; + void initializeIdentity(); + void resetConnectionStatus(); + void recordMediaError(const std::string& error); + void setState(QuicEdgeServiceState state, const std::string& error = {}); + bool waitForStop(std::chrono::milliseconds duration); + std::chrono::milliseconds nextBackoff(std::chrono::milliseconds current); + std::chrono::milliseconds jittered(std::chrono::milliseconds base); + std::uint64_t nextSessionEpoch(); + + config::QuicEdgeConfig config_; + std::unique_ptr transport_; + media::MediaSourceHub* media_hub_{nullptr}; + bool using_default_transport_{false}; + bool using_global_media_hub_{false}; + SourceRegistrar source_registrar_; + + std::mutex lifecycle_mutex_; + std::mutex control_send_mutex_; + mutable std::mutex mutex_; + std::condition_variable stop_cv_; + std::condition_variable media_stop_cv_; + QuicEdgeServiceState state_{QuicEdgeServiceState::UNINITIALIZED}; + std::string last_error_; + std::string last_media_error_; + QuicEdgeStats stats_; + bool registered_{false}; + std::string session_id_; + std::string observed_source_ip_; + std::uint64_t heartbeat_sequence_{0}; + std::uint64_t outstanding_heartbeat_sequence_{0}; + std::uint64_t last_heartbeat_ack_unix_ms_{0}; + std::size_t active_media_tracks_{0}; + bool stop_requested_{false}; + bool media_stop_requested_{false}; + bool media_connection_failed_{false}; + std::string media_connection_error_; + std::thread worker_; + std::thread media_worker_; + + std::string node_id_; + std::string boot_id_; + std::string software_version_; + DatagramPacketizer packetizer_; + std::uint64_t session_epoch_{0}; + std::unordered_map next_wire_frame_sequence_; + std::uint64_t control_message_sequence_{0}; + std::uint64_t inbound_message_sequence_{0}; + bool has_inbound_message_sequence_{false}; + std::uint32_t effective_heartbeat_interval_ms_{0}; + std::chrono::steady_clock::time_point heartbeat_deadline_{}; + std::chrono::steady_clock::time_point next_heartbeat_{}; + std::chrono::steady_clock::time_point next_media_source_retry_{}; + bool media_session_announced_{false}; +}; + +} // namespace cmvr::quic_edge + +#endif // CMVR_ES_QUIC_EDGE_SERVICE_H diff --git a/cmvr-es/service/quic_edge/include/quic_edge_types.h b/cmvr-es/service/quic_edge/include/quic_edge_types.h new file mode 100644 index 00000000..36fda822 --- /dev/null +++ b/cmvr-es/service/quic_edge/include/quic_edge_types.h @@ -0,0 +1,118 @@ +#ifndef CMVR_ES_QUIC_EDGE_TYPES_H +#define CMVR_ES_QUIC_EDGE_TYPES_H + +#include +#include +#include +#include + +#include "common/media/media_frame.h" + +namespace cmvr::quic_edge { + +constexpr std::uint32_t kDatagramMagic = 0x434d5144U; // "CMQD" +constexpr std::uint8_t kProtocolVersion = 1U; +constexpr std::size_t kDatagramHeaderBytes = 64U; + +enum DatagramFlag : std::uint16_t { + DATAGRAM_FLAG_NONE = 0, + DATAGRAM_FLAG_KEY_FRAME = 1U << 0U, + DATAGRAM_FLAG_RESERVED_1 = 1U << 1U, + DATAGRAM_FLAG_DISCONTINUITY = 1U << 2U, +}; + +// All integer fields are serialized in network byte order. codec_generation +// is a compact token for the full 64-bit MediaSourceHub descriptor generation +// announced on the reliable control stream. +struct DatagramHeader { + std::uint8_t protocol_version{kProtocolVersion}; + media::MediaKind kind{media::MediaKind::UNKNOWN}; + std::uint16_t flags{DATAGRAM_FLAG_NONE}; + std::uint16_t fragment_index{0}; + std::uint16_t fragment_count{0}; + std::uint16_t payload_size{0}; + std::uint32_t track_id{0}; + std::uint32_t codec_generation{0}; + std::uint64_t session_epoch{0}; + std::uint64_t packet_sequence{0}; + std::uint64_t frame_sequence{0}; + std::uint64_t capture_timestamp_us{0}; + std::uint32_t frame_size{0}; + std::uint32_t fragment_offset{0}; +}; + +struct DatagramPacket { + DatagramHeader header; + std::vector bytes; +}; + +struct MediaTrackDescription { + std::uint32_t track_id{0}; + std::uint32_t codec_generation_token{0}; + std::string source_track_id; + std::string source_id; + media::MediaKind kind{media::MediaKind::UNKNOWN}; + media::Codec codec{media::Codec::UNKNOWN}; + media::PayloadFormat payload_format{media::PayloadFormat::UNKNOWN}; + std::uint64_t codec_generation{0}; + std::uint32_t width{0}; + std::uint32_t height{0}; + std::uint32_t nominal_rate{0}; + std::uint32_t sample_rate{0}; + std::uint32_t channels{0}; + std::vector codec_config; + + bool operator==(const MediaTrackDescription& other) const + { + return track_id == other.track_id && + codec_generation_token == other.codec_generation_token && + source_track_id == other.source_track_id && + source_id == other.source_id && + kind == other.kind && + codec == other.codec && + payload_format == other.payload_format && + codec_generation == other.codec_generation && + width == other.width && + height == other.height && + nominal_rate == other.nominal_rate && + sample_rate == other.sample_rate && + channels == other.channels && + codec_config == other.codec_config; + } + + bool operator!=(const MediaTrackDescription& other) const + { + return !(*this == other); + } +}; + +std::uint32_t descriptorGenerationToken(std::uint64_t generation); +const char* codecName(media::Codec codec); +const char* payloadFormatName(media::PayloadFormat format); + +inline MediaTrackDescription describeTrack( + const std::uint32_t wire_track_id, + const media::TrackDescriptor& descriptor) +{ + MediaTrackDescription description; + description.track_id = wire_track_id; + description.codec_generation_token = + descriptorGenerationToken(descriptor.generation); + description.source_track_id = descriptor.id; + description.source_id = descriptor.source_id; + description.kind = descriptor.kind; + description.codec = descriptor.codec; + description.payload_format = descriptor.payload_format; + description.codec_generation = descriptor.generation; + description.width = descriptor.width; + description.height = descriptor.height; + description.nominal_rate = descriptor.nominal_rate; + description.sample_rate = descriptor.sample_rate; + description.channels = descriptor.channels; + description.codec_config = descriptor.codec_config; + return description; +} + +} // namespace cmvr::quic_edge + +#endif // CMVR_ES_QUIC_EDGE_TYPES_H diff --git a/cmvr-es/service/quic_edge/include/quic_transport.h b/cmvr-es/service/quic_edge/include/quic_transport.h new file mode 100644 index 00000000..44c3c1b8 --- /dev/null +++ b/cmvr-es/service/quic_edge/include/quic_transport.h @@ -0,0 +1,74 @@ +#ifndef CMVR_ES_QUIC_EDGE_QUIC_TRANSPORT_H +#define CMVR_ES_QUIC_EDGE_QUIC_TRANSPORT_H + +#include +#include +#include +#include +#include +#include + +#include "cmvr/config/quic_edge_config/quic_edge_config.pb.h" +#include "service/quic_edge/include/quic_edge_types.h" + +namespace cmvr::quic_edge { + +enum class TransportSendResult { + QUEUED, + WOULD_BLOCK, + DISCONNECTED, + ERROR, +}; + +enum class TransportReceiveResult { + DATA, + TIMEOUT, + DISCONNECTED, + ERROR, +}; + +class QuicTransport { +public: + virtual ~QuicTransport() = default; + + virtual bool connect(const config::QuicEdgeConfig& config, + std::chrono::milliseconds timeout, + std::string* error) = 0; + virtual void disconnect() = 0; + virtual bool isConnected() const = 0; + virtual std::size_t maximumDatagramBytes() const = 0; + virtual std::size_t maximumDatagramBatchPackets() const = 0; + + // QUEUED means ownership was accepted. WOULD_BLOCK and synchronous ERROR + // mean no bytes were accepted, so the caller may safely retry the complete + // framed message. DISCONNECTED means the connection is no longer usable. + virtual TransportSendResult sendControl( + std::vector framed_message, + std::string* error) = 0; + + // Returns one raw byte chunk from the reliable control stream. QUIC stream + // callback boundaries are not message boundaries; callers must feed all + // DATA chunks into their framing decoder in order. + // + // The default implementation reports DISCONNECTED so existing transports + // that only support sending remain source-compatible. + virtual TransportReceiveResult receiveControl( + std::vector* chunk, + std::chrono::milliseconds timeout, + std::string* error); + + // Local admission is atomic for the complete batch. A native QUIC API may + // still fail after accepting earlier fragments; in that case the transport + // must reset the connection so the receiver discards the partial session. + virtual TransportSendResult sendDatagramBatch( + std::vector packets, + std::string* error) = 0; +}; + +bool hasCompiledMsQuicSupport(); +std::unique_ptr createDefaultQuicTransport( + std::size_t send_queue_depth); + +} // namespace cmvr::quic_edge + +#endif // CMVR_ES_QUIC_EDGE_QUIC_TRANSPORT_H diff --git a/cmvr-es/service/quic_edge/src/control_framing.cpp b/cmvr-es/service/quic_edge/src/control_framing.cpp new file mode 100644 index 00000000..a47f49c0 --- /dev/null +++ b/cmvr-es/service/quic_edge/src/control_framing.cpp @@ -0,0 +1,115 @@ +#include "service/quic_edge/include/control_framing.h" + +#include +#include + +namespace cmvr::quic_edge { +namespace { + +void setError(std::string* error, const std::string& message) +{ + if (error) { + *error = message; + } +} + +std::uint32_t readLength(const std::vector& buffer) +{ + return (static_cast(buffer[0]) << 24U) | + (static_cast(buffer[1]) << 16U) | + (static_cast(buffer[2]) << 8U) | + static_cast(buffer[3]); +} + +} // namespace + +bool ControlFrameEncoder::encode(const std::uint8_t* payload, + const std::size_t payload_size, + const std::size_t maximum_payload_size, + std::vector* framed, + std::string* error) +{ + if (!framed) { + setError(error, "control-frame output is null"); + return false; + } + framed->clear(); + if (!payload || payload_size == 0U) { + setError(error, "control-frame payload is empty"); + return false; + } + if (payload_size > maximum_payload_size || + payload_size > std::numeric_limits::max()) { + setError(error, "control-frame payload exceeds configured limit"); + return false; + } + + const auto length = static_cast(payload_size); + framed->resize(4U + payload_size); + (*framed)[0] = static_cast(length >> 24U); + (*framed)[1] = static_cast(length >> 16U); + (*framed)[2] = static_cast(length >> 8U); + (*framed)[3] = static_cast(length); + std::copy_n(payload, payload_size, framed->data() + 4U); + return true; +} + +ControlFrameDecoder::ControlFrameDecoder(const std::size_t maximum_payload_size) + : maximum_payload_size_(maximum_payload_size) +{ + buffer_.reserve(std::min(maximum_payload_size + 4U, 4096U)); +} + +bool ControlFrameDecoder::push(const std::uint8_t* data, + const std::size_t size, + std::vector>* decoded_frames, + std::string* error) +{ + if (!decoded_frames) { + setError(error, "decoded control-frame output is null"); + return false; + } + if (size != 0U && !data) { + setError(error, "control-frame input is null"); + return false; + } + + std::size_t consumed = 0U; + while (consumed < size) { + if (buffer_.size() < 4U) { + const std::size_t prefix_bytes = + std::min(4U - buffer_.size(), size - consumed); + buffer_.insert(buffer_.end(), data + consumed, + data + consumed + prefix_bytes); + consumed += prefix_bytes; + if (buffer_.size() < 4U) { + break; + } + const std::uint32_t declared = readLength(buffer_); + if (declared == 0U || declared > maximum_payload_size_) { + reset(); + setError(error, "control-frame declared length is invalid"); + return false; + } + } + + const std::size_t total_size = 4U + readLength(buffer_); + const std::size_t bytes_needed = total_size - buffer_.size(); + const std::size_t copied = + std::min(bytes_needed, size - consumed); + buffer_.insert(buffer_.end(), data + consumed, data + consumed + copied); + consumed += copied; + if (buffer_.size() == total_size) { + decoded_frames->emplace_back(buffer_.begin() + 4, buffer_.end()); + buffer_.clear(); + } + } + return true; +} + +void ControlFrameDecoder::reset() +{ + buffer_.clear(); +} + +} // namespace cmvr::quic_edge diff --git a/cmvr-es/service/quic_edge/src/datagram_packetizer.cpp b/cmvr-es/service/quic_edge/src/datagram_packetizer.cpp new file mode 100644 index 00000000..6e8854d8 --- /dev/null +++ b/cmvr-es/service/quic_edge/src/datagram_packetizer.cpp @@ -0,0 +1,270 @@ +#include "service/quic_edge/include/datagram_packetizer.h" + +#include +#include + +namespace cmvr::quic_edge { +namespace { + +void setError(std::string* error, const std::string& message) +{ + if (error) { + *error = message; + } +} + +void writeU16(std::vector* out, const std::size_t offset, + const std::uint16_t value) +{ + (*out)[offset] = static_cast(value >> 8U); + (*out)[offset + 1U] = static_cast(value); +} + +void writeU32(std::vector* out, const std::size_t offset, + const std::uint32_t value) +{ + (*out)[offset] = static_cast(value >> 24U); + (*out)[offset + 1U] = static_cast(value >> 16U); + (*out)[offset + 2U] = static_cast(value >> 8U); + (*out)[offset + 3U] = static_cast(value); +} + +void writeU64(std::vector* out, const std::size_t offset, + const std::uint64_t value) +{ + for (std::size_t i = 0; i < 8U; ++i) { + (*out)[offset + i] = static_cast(value >> (56U - i * 8U)); + } +} + +std::uint16_t readU16(const std::uint8_t* data, const std::size_t offset) +{ + return static_cast( + (static_cast(data[offset]) << 8U) | + static_cast(data[offset + 1U])); +} + +std::uint32_t readU32(const std::uint8_t* data, const std::size_t offset) +{ + return (static_cast(data[offset]) << 24U) | + (static_cast(data[offset + 1U]) << 16U) | + (static_cast(data[offset + 2U]) << 8U) | + static_cast(data[offset + 3U]); +} + +std::uint64_t readU64(const std::uint8_t* data, const std::size_t offset) +{ + std::uint64_t value = 0; + for (std::size_t i = 0; i < 8U; ++i) { + value = (value << 8U) | static_cast(data[offset + i]); + } + return value; +} + +bool validKind(const media::MediaKind kind) +{ + return kind == media::MediaKind::VIDEO || kind == media::MediaKind::AUDIO; +} + +} // namespace + +DatagramPacketizer::DatagramPacketizer(const std::uint64_t initial_packet_sequence) + : next_packet_sequence_(initial_packet_sequence) +{ +} + +bool DatagramPacketizer::packetize(const media::MediaFrame& frame, + const std::uint32_t wire_track_id, + const std::uint32_t codec_generation_token, + const std::uint64_t session_epoch, + const std::uint64_t wire_frame_sequence, + const bool force_discontinuity, + const std::size_t maximum_datagram_bytes, + std::vector* packets, + std::string* error) +{ + if (!packets) { + setError(error, "packet output is null"); + return false; + } + packets->clear(); + if (wire_track_id == 0U) { + setError(error, "track_id must be non-zero"); + return false; + } + if (!frame.descriptor || !validKind(frame.descriptor->kind)) { + setError(error, "unsupported media kind"); + return false; + } + if (codec_generation_token == 0U) { + setError(error, "codec generation token must be non-zero"); + return false; + } + if (session_epoch == 0U) { + setError(error, "session_epoch must be non-zero"); + return false; + } + if (wire_frame_sequence == 0U) { + setError(error, "wire_frame_sequence must be non-zero"); + return false; + } + if (frame.empty()) { + setError(error, "media frame payload is empty"); + return false; + } + if (frame.size() > std::numeric_limits::max()) { + setError(error, "media frame is larger than the v1 frame_size field"); + return false; + } + if (maximum_datagram_bytes <= kDatagramHeaderBytes) { + setError(error, "maximum DATAGRAM size does not leave room for payload"); + return false; + } + + const std::size_t payload_capacity = std::min( + maximum_datagram_bytes - kDatagramHeaderBytes, + std::numeric_limits::max()); + const std::size_t fragment_count = + (frame.size() + payload_capacity - 1U) / payload_capacity; + if (fragment_count == 0U || + fragment_count > std::numeric_limits::max()) { + setError(error, "media frame requires too many DATAGRAM fragments"); + return false; + } + if (next_packet_sequence_ > + std::numeric_limits::max() - fragment_count) { + setError(error, "DATAGRAM packet sequence exhausted"); + return false; + } + + std::uint16_t flags = DATAGRAM_FLAG_NONE; + if (frame.key_frame) { + flags |= DATAGRAM_FLAG_KEY_FRAME; + } + if (frame.discontinuity || force_discontinuity) { + flags |= DATAGRAM_FLAG_DISCONTINUITY; + } + + std::vector encoded_packets; + encoded_packets.reserve(fragment_count); + for (std::size_t i = 0; i < fragment_count; ++i) { + const std::size_t offset = i * payload_capacity; + const std::size_t payload_size = + std::min(payload_capacity, frame.size() - offset); + + DatagramPacket packet; + packet.header.protocol_version = kProtocolVersion; + packet.header.kind = frame.descriptor->kind; + packet.header.flags = flags; + packet.header.fragment_index = static_cast(i); + packet.header.fragment_count = static_cast(fragment_count); + packet.header.payload_size = static_cast(payload_size); + packet.header.track_id = wire_track_id; + packet.header.codec_generation = codec_generation_token; + packet.header.session_epoch = session_epoch; + packet.header.packet_sequence = next_packet_sequence_ + i; + packet.header.frame_sequence = wire_frame_sequence; + packet.header.capture_timestamp_us = frame.capture_time_ns / 1000U; + packet.header.frame_size = static_cast(frame.size()); + packet.header.fragment_offset = static_cast(offset); + + packet.bytes.resize(kDatagramHeaderBytes + payload_size); + writeU32(&packet.bytes, 0U, kDatagramMagic); + packet.bytes[4U] = packet.header.protocol_version; + packet.bytes[5U] = static_cast(packet.header.kind); + writeU16(&packet.bytes, 6U, packet.header.flags); + writeU16(&packet.bytes, 8U, static_cast(kDatagramHeaderBytes)); + writeU16(&packet.bytes, 10U, packet.header.fragment_index); + writeU16(&packet.bytes, 12U, packet.header.fragment_count); + writeU16(&packet.bytes, 14U, packet.header.payload_size); + writeU32(&packet.bytes, 16U, packet.header.track_id); + writeU32(&packet.bytes, 20U, packet.header.codec_generation); + writeU64(&packet.bytes, 24U, packet.header.session_epoch); + writeU64(&packet.bytes, 32U, packet.header.packet_sequence); + writeU64(&packet.bytes, 40U, packet.header.frame_sequence); + writeU64(&packet.bytes, 48U, packet.header.capture_timestamp_us); + writeU32(&packet.bytes, 56U, packet.header.frame_size); + writeU32(&packet.bytes, 60U, packet.header.fragment_offset); + std::copy_n(frame.data() + offset, payload_size, + packet.bytes.data() + kDatagramHeaderBytes); + encoded_packets.push_back(std::move(packet)); + } + + next_packet_sequence_ += fragment_count; + *packets = std::move(encoded_packets); + return true; +} + +bool DatagramPacketizer::decodeHeader(const std::uint8_t* data, + const std::size_t size, + DatagramHeader* header, + std::string* error) +{ + if (!data || !header) { + setError(error, "DATAGRAM input or header output is null"); + return false; + } + if (size < kDatagramHeaderBytes) { + setError(error, "DATAGRAM is shorter than the v1 header"); + return false; + } + if (readU32(data, 0U) != kDatagramMagic) { + setError(error, "DATAGRAM magic mismatch"); + return false; + } + if (data[4U] != kProtocolVersion) { + setError(error, "unsupported DATAGRAM protocol version"); + return false; + } + const auto kind = static_cast(data[5U]); + if (!validKind(kind)) { + setError(error, "unsupported DATAGRAM media kind"); + return false; + } + if (readU16(data, 8U) != kDatagramHeaderBytes) { + setError(error, "unsupported DATAGRAM header size"); + return false; + } + + DatagramHeader decoded; + decoded.protocol_version = data[4U]; + decoded.kind = kind; + decoded.flags = readU16(data, 6U); + decoded.fragment_index = readU16(data, 10U); + decoded.fragment_count = readU16(data, 12U); + decoded.payload_size = readU16(data, 14U); + decoded.track_id = readU32(data, 16U); + decoded.codec_generation = readU32(data, 20U); + decoded.session_epoch = readU64(data, 24U); + decoded.packet_sequence = readU64(data, 32U); + decoded.frame_sequence = readU64(data, 40U); + decoded.capture_timestamp_us = readU64(data, 48U); + decoded.frame_size = readU32(data, 56U); + decoded.fragment_offset = readU32(data, 60U); + + if (decoded.track_id == 0U || decoded.codec_generation == 0U || + decoded.session_epoch == 0U) { + setError(error, "DATAGRAM has a zero track, generation, or session identifier"); + return false; + } + if (decoded.fragment_count == 0U || + decoded.fragment_index >= decoded.fragment_count) { + setError(error, "DATAGRAM fragment index/count is invalid"); + return false; + } + if (decoded.payload_size != size - kDatagramHeaderBytes) { + setError(error, "DATAGRAM payload size does not match packet length"); + return false; + } + if (decoded.frame_size == 0U || decoded.payload_size == 0U || + decoded.fragment_offset > decoded.frame_size || + decoded.payload_size > decoded.frame_size - decoded.fragment_offset) { + setError(error, "DATAGRAM fragment is outside the declared frame"); + return false; + } + + *header = decoded; + return true; +} + +} // namespace cmvr::quic_edge diff --git a/cmvr-es/service/quic_edge/src/msquic_transport.cpp b/cmvr-es/service/quic_edge/src/msquic_transport.cpp new file mode 100644 index 00000000..c8928058 --- /dev/null +++ b/cmvr-es/service/quic_edge/src/msquic_transport.cpp @@ -0,0 +1,852 @@ +#include "service/quic_edge/include/quic_transport.h" + +#ifdef CMVR_HAS_MSQUIC + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cmvr::quic_edge { +namespace { + +std::string statusText(const char* operation, const QUIC_STATUS status) +{ + std::ostringstream output; + output << operation << " failed with QUIC_STATUS 0x" << std::hex + << static_cast(status); + return output.str(); +} + +constexpr std::size_t kControlFramePrefixBytes = sizeof(std::uint32_t); +constexpr std::size_t kMinimumControlReceiveQueueBytes = 64U * 1024U; +constexpr std::size_t kMaximumControlReceiveQueueBytes = + 32U * 1024U * 1024U + 2U * kControlFramePrefixBytes; +constexpr std::size_t kMaximumControlReceiveQueueChunks = 4096U; +constexpr std::size_t kMaximumReservedControlSends = 8U; +constexpr std::uint64_t kMaximumHeartbeatIntervalMs = 60ULL * 60ULL * 1000ULL; + +std::size_t reservedControlSends(const std::size_t queue_depth) +{ + if (queue_depth <= 1U) return 0U; + return std::min( + kMaximumReservedControlSends, + std::max(1U, queue_depth / 16U)); +} + +std::size_t controlReceiveQueueBytes(const config::QuicEdgeConfig& config) +{ + // Keep enough room for two maximum-sized framed control messages while + // enforcing a transport-level hard bound independent of peer behavior. + const std::uint64_t desired = + 2ULL * (static_cast( + config.maximum_control_frame_bytes()) + + kControlFramePrefixBytes); + return static_cast(std::clamp( + desired, kMinimumControlReceiveQueueBytes, + kMaximumControlReceiveQueueBytes)); +} + +class MsQuicTransport final : public QuicTransport { +public: + explicit MsQuicTransport(const std::size_t send_queue_depth) + : send_queue_depth_(std::max(1U, send_queue_depth)), + reserved_control_sends_(reservedControlSends(send_queue_depth_)) + { + } + + ~MsQuicTransport() override + { + disconnect(); + if (registration_ && api_) api_->RegistrationClose(registration_); + if (api_) MsQuicClose(api_); + } + + bool connect(const config::QuicEdgeConfig& config, + const std::chrono::milliseconds timeout, + std::string* error) override + { + disconnect(); + QUIC_STATUS status = QUIC_STATUS_SUCCESS; + { + // Configuration creation and connection publication are one API + // operation. A concurrent disconnect can therefore never close a + // just-created configuration before ConnectionStart consumes it. + std::lock_guard operation_lock(api_operation_mutex_); + if (!openApi(error) || !openConfiguration(config, error)) { + return false; + } + { + std::lock_guard lock(mutex_); + shutdown_complete_ = false; + connected_ = false; + datagram_send_enabled_ = false; + maximum_datagram_bytes_ = 0U; + control_stream_start_completed_ = false; + control_stream_started_ = false; + control_stream_shutdown_complete_ = false; + control_stream_receive_closed_ = true; + control_receive_queue_.clear(); + control_receive_queue_bytes_ = 0U; + maximum_control_receive_queue_bytes_ = + controlReceiveQueueBytes(config); + control_receive_error_.clear(); + connect_error_.clear(); + } + HQUIC opened_connection = nullptr; + status = api_->ConnectionOpen( + registration_, &MsQuicTransport::connectionCallback, this, + &opened_connection); + if (QUIC_SUCCEEDED(status)) { + { + std::lock_guard state_lock(mutex_); + connection_ = opened_connection; + } + const std::string& server_name = + config.tls().allow_insecure() || config.tls().server_name().empty() + ? config.server_host() : config.tls().server_name(); + status = api_->ConnectionStart( + opened_connection, configuration_, QUIC_ADDRESS_FAMILY_UNSPEC, + server_name.c_str(), + static_cast(config.server_port())); + } + } + if (QUIC_FAILED(status)) { + setError(error, statusText("ConnectionOpen/Start", status)); + closeConnectionHandles(); + return false; + } + + { + std::unique_lock lock(mutex_); + if (!condition_.wait_for(lock, timeout, [this]() { + return connected_ || shutdown_complete_; + }) || !connected_) { + const std::string message = connect_error_.empty() + ? "MsQuic connect timed out" : connect_error_; + lock.unlock(); + disconnect(); + setError(error, message); + return false; + } + } + + bool connection_available = false; + { + std::lock_guard operation_lock(api_operation_mutex_); + HQUIC connection = nullptr; + HQUIC opened_stream = nullptr; + { + std::lock_guard state_lock(mutex_); + connection_available = connected_ && connection_; + connection = connection_; + } + if (connection_available) { + status = api_->StreamOpen( + connection, QUIC_STREAM_OPEN_FLAG_NONE, + &MsQuicTransport::streamCallback, this, &opened_stream); + if (QUIC_SUCCEEDED(status)) { + { + std::lock_guard state_lock(mutex_); + control_stream_ = opened_stream; + control_stream_start_completed_ = false; + control_stream_started_ = false; + control_stream_shutdown_complete_ = false; + control_stream_receive_closed_ = false; + } + status = api_->StreamStart( + opened_stream, QUIC_STREAM_START_FLAG_IMMEDIATE); + } + } + } + if (!connection_available) { + setError(error, "QUIC connection closed before control stream setup"); + disconnect(); + return false; + } + if (QUIC_FAILED(status)) { + { + std::lock_guard lock(mutex_); + control_stream_start_completed_ = true; + control_stream_started_ = false; + } + setError(error, statusText("control StreamOpen/Start", status)); + disconnect(); + return false; + } + + { + std::unique_lock lock(mutex_); + const bool completed = condition_.wait_for(lock, timeout, [this]() { + return control_stream_start_completed_ || !connected_; + }); + if (!completed || !connected_ || !control_stream_started_) { + const std::string message = connect_error_.empty() + ? "QUIC control stream start did not complete successfully" + : connect_error_; + lock.unlock(); + disconnect(); + setError(error, message); + return false; + } + } + return true; + } + + void disconnect() override + { + HQUIC connection = nullptr; + HQUIC stream = nullptr; + bool stream_started = false; + { + // Resolve and use the handle while holding the API-operation lock. + // This prevents a concurrent closeConnectionHandles() call from + // closing the handle between the state read and ConnectionShutdown. + std::lock_guard operation_lock(api_operation_mutex_); + { + std::lock_guard lock(mutex_); + connection = connection_; + stream = control_stream_; + stream_started = control_stream_started_; + connected_ = false; + control_stream_receive_closed_ = true; + ++control_receive_epoch_; + condition_.notify_all(); + } + if (stream && stream_started && api_) { + const auto flags = static_cast( + QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND | + QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE | + QUIC_STREAM_SHUTDOWN_FLAG_IMMEDIATE); + (void)api_->StreamShutdown(stream, flags, 0); + } + if (connection && api_) { + api_->ConnectionShutdown( + connection, QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0); + } + } + if (stream && stream_started && api_) { + std::unique_lock lock(mutex_); + // A started stream handle may only be closed after MsQuic delivers + // SHUTDOWN_COMPLETE. IMMEDIATE schedules that completion without a + // graceful network wait, but it is not guaranteed to run inline. + condition_.wait(lock, [this, stream]() { + return control_stream_ != stream || + control_stream_shutdown_complete_; + }); + } + closeConnectionHandles(); + } + + bool isConnected() const override + { + std::lock_guard lock(mutex_); + return connected_; + } + + std::size_t maximumDatagramBytes() const override + { + std::lock_guard lock(mutex_); + return maximum_datagram_bytes_; + } + + std::size_t maximumDatagramBatchPackets() const override + { + // Static empty-queue limit. sendDatagramBatch performs the real-time + // admission check under mutex and may still return WOULD_BLOCK. + return send_queue_depth_ - reserved_control_sends_; + } + + TransportSendResult sendControl(std::vector message, + std::string* error) override + { + std::lock_guard operation_lock(api_operation_mutex_); + HQUIC stream = nullptr; + { + std::lock_guard lock(mutex_); + if (!connected_ || !control_stream_) return TransportSendResult::DISCONNECTED; + if (pending_send_count_ >= send_queue_depth_) { + return TransportSendResult::WOULD_BLOCK; + } + ++pending_send_count_; + stream = control_stream_; + } + auto* context = new SendContext(this, std::move(message)); + registerSend(context); + const QUIC_STATUS status = api_->StreamSend( + stream, &context->buffer, 1, QUIC_SEND_FLAG_NONE, context); + if (QUIC_FAILED(status)) { + finishSend(context); + setError(error, statusText("StreamSend", status)); + return TransportSendResult::ERROR; + } + return TransportSendResult::QUEUED; + } + + TransportReceiveResult receiveControl( + std::vector* chunk, + const std::chrono::milliseconds timeout, + std::string* error) override + { + if (!chunk) { + setError(error, "control receive output must not be null"); + return TransportReceiveResult::ERROR; + } + chunk->clear(); + if (error) error->clear(); + if (timeout.count() < 0) { + setError(error, "control receive timeout must not be negative"); + return TransportReceiveResult::ERROR; + } + + std::unique_lock lock(mutex_); + const std::uint64_t receive_epoch = control_receive_epoch_; + const auto ready = [this, receive_epoch]() { + return !control_receive_error_.empty() || + !control_receive_queue_.empty() || + control_stream_receive_closed_ || !connected_ || + control_receive_epoch_ != receive_epoch; + }; + if (!ready()) { + bool signaled = false; + if (timeout == std::chrono::milliseconds::max()) { + condition_.wait(lock, ready); + signaled = true; + } else { + signaled = condition_.wait_for(lock, timeout, ready); + } + if (!signaled) return TransportReceiveResult::TIMEOUT; + } + + // A disconnect/reconnect boundary invalidates the old stream decoder + // even if a new connection became ready before this waiter ran. + if (control_receive_epoch_ != receive_epoch) { + setError(error, "QUIC control stream session changed"); + return TransportReceiveResult::DISCONNECTED; + } + if (!control_receive_error_.empty()) { + setError(error, control_receive_error_); + return TransportReceiveResult::ERROR; + } + // Preserve bytes already delivered by MsQuic even when shutdown raced + // with the consumer; report DISCONNECTED only after draining them. + if (!control_receive_queue_.empty()) { + *chunk = std::move(control_receive_queue_.front()); + control_receive_queue_.pop_front(); + control_receive_queue_bytes_ -= chunk->size(); + return TransportReceiveResult::DATA; + } + setError(error, connect_error_.empty() + ? "QUIC control stream is disconnected" : connect_error_); + return TransportReceiveResult::DISCONNECTED; + } + + TransportSendResult sendDatagramBatch( + std::vector packets, + std::string* error) override + { + std::lock_guard operation_lock(api_operation_mutex_); + if (packets.empty()) { + setError(error, "empty MsQuic DATAGRAM batch"); + return TransportSendResult::ERROR; + } + HQUIC connection = nullptr; + { + std::lock_guard lock(mutex_); + if (!connected_ || !connection_) return TransportSendResult::DISCONNECTED; + if (!datagram_send_enabled_) { + setError(error, "peer/path disabled QUIC DATAGRAM sending"); + return TransportSendResult::ERROR; + } + for (const auto& packet : packets) { + if (packet.bytes.empty() || + packet.bytes.size() > maximum_datagram_bytes_) { + setError(error, + "QUIC DATAGRAM exceeds the current path limit"); + return TransportSendResult::ERROR; + } + } + const std::size_t datagram_capacity = + send_queue_depth_ - reserved_control_sends_; + if (pending_send_count_ >= datagram_capacity || + packets.size() > datagram_capacity - pending_send_count_) { + return TransportSendResult::WOULD_BLOCK; + } + pending_send_count_ += packets.size(); + connection = connection_; + } + + std::size_t submitted = 0; + for (auto& packet : packets) { + auto* context = new SendContext(this, std::move(packet.bytes)); + registerSend(context); + const QUIC_STATUS status = api_->DatagramSend( + connection, &context->buffer, 1, QUIC_SEND_FLAG_NONE, context); + if (QUIC_FAILED(status)) { + finishSend(context); + releaseReservedSends(packets.size() - submitted - 1U); + setError(error, statusText("DatagramSend", status)); + // A submission error after earlier fragments is terminal. The + // reconnect/session epoch prevents receivers from combining a + // partial old frame with new traffic. + api_->ConnectionShutdown(connection, + QUIC_CONNECTION_SHUTDOWN_FLAG_SILENT, 1); + return TransportSendResult::ERROR; + } + ++submitted; + } + return TransportSendResult::QUEUED; + } + +private: + struct SendContext { + SendContext(MsQuicTransport* value_owner, std::vector value) + : owner(value_owner), bytes(std::move(value)) + { + buffer.Buffer = bytes.data(); + buffer.Length = static_cast(bytes.size()); + } + MsQuicTransport* owner; + std::vector bytes; + QUIC_BUFFER buffer{}; + }; + + static QUIC_STATUS QUIC_API connectionCallback( + HQUIC connection, void* context, QUIC_CONNECTION_EVENT* event) + { + auto* self = static_cast(context); + switch (event->Type) { + case QUIC_CONNECTION_EVENT_CONNECTED: { + std::lock_guard lock(self->mutex_); + if (connection != self->connection_) break; + self->connected_ = true; + self->condition_.notify_all(); + break; + } + case QUIC_CONNECTION_EVENT_SHUTDOWN_INITIATED_BY_TRANSPORT: { + std::lock_guard lock(self->mutex_); + if (connection != self->connection_) break; + self->connected_ = false; + self->control_stream_receive_closed_ = true; + self->connect_error_ = statusText( + "peer/transport shutdown", + event->SHUTDOWN_INITIATED_BY_TRANSPORT.Status); + self->condition_.notify_all(); + break; + } + case QUIC_CONNECTION_EVENT_SHUTDOWN_INITIATED_BY_PEER: { + std::lock_guard lock(self->mutex_); + if (connection != self->connection_) break; + self->connected_ = false; + self->control_stream_receive_closed_ = true; + self->connect_error_ = "peer closed the QUIC connection"; + self->condition_.notify_all(); + break; + } + case QUIC_CONNECTION_EVENT_SHUTDOWN_COMPLETE: { + std::lock_guard lock(self->mutex_); + if (connection != self->connection_) break; + self->connected_ = false; + self->control_stream_receive_closed_ = true; + self->shutdown_complete_ = true; + self->condition_.notify_all(); + break; + } + case QUIC_CONNECTION_EVENT_DATAGRAM_SEND_STATE_CHANGED: + switch (event->DATAGRAM_SEND_STATE_CHANGED.State) { + case QUIC_DATAGRAM_SEND_LOST_DISCARDED: + case QUIC_DATAGRAM_SEND_ACKNOWLEDGED: + case QUIC_DATAGRAM_SEND_ACKNOWLEDGED_SPURIOUS: + case QUIC_DATAGRAM_SEND_CANCELED: + self->finishSend(static_cast( + event->DATAGRAM_SEND_STATE_CHANGED.ClientContext)); + break; + default: + break; + } + break; + case QUIC_CONNECTION_EVENT_DATAGRAM_STATE_CHANGED: { + std::lock_guard lock(self->mutex_); + if (connection != self->connection_) break; + self->datagram_send_enabled_ = + event->DATAGRAM_STATE_CHANGED.SendEnabled != FALSE; + self->maximum_datagram_bytes_ = self->datagram_send_enabled_ + ? event->DATAGRAM_STATE_CHANGED.MaxSendLength : 0U; + self->condition_.notify_all(); + break; + } + default: + break; + } + return QUIC_STATUS_SUCCESS; + } + + static QUIC_STATUS QUIC_API streamCallback( + HQUIC stream, void* context, QUIC_STREAM_EVENT* event) + { + auto* self = static_cast(context); + switch (event->Type) { + case QUIC_STREAM_EVENT_START_COMPLETE: { + std::lock_guard lock(self->mutex_); + if (stream != self->control_stream_) break; + self->control_stream_start_completed_ = true; + self->control_stream_started_ = + QUIC_SUCCEEDED(event->START_COMPLETE.Status); + if (!self->control_stream_started_) { + self->control_stream_receive_closed_ = true; + self->connect_error_ = statusText( + "control stream start", event->START_COMPLETE.Status); + } + self->condition_.notify_all(); + break; + } + case QUIC_STREAM_EVENT_RECEIVE: + return self->handleControlReceive(stream, event); + case QUIC_STREAM_EVENT_SEND_COMPLETE: + if (event->SEND_COMPLETE.Canceled != FALSE) { + std::lock_guard lock(self->mutex_); + if (stream == self->control_stream_ && self->connected_) { + if (self->connect_error_.empty()) { + self->connect_error_ = + "QUIC control stream send was canceled"; + } + self->control_stream_receive_closed_ = true; + self->connected_ = false; + self->condition_.notify_all(); + } + } + self->finishSend(static_cast( + event->SEND_COMPLETE.ClientContext)); + break; + case QUIC_STREAM_EVENT_PEER_RECEIVE_ABORTED: { + std::lock_guard lock(self->mutex_); + if (stream != self->control_stream_ || !self->connected_) break; + if (self->connect_error_.empty()) { + self->connect_error_ = + "peer aborted the QUIC control stream receive direction"; + } + self->control_stream_receive_closed_ = true; + self->connected_ = false; + self->condition_.notify_all(); + break; + } + case QUIC_STREAM_EVENT_PEER_SEND_SHUTDOWN: + case QUIC_STREAM_EVENT_PEER_SEND_ABORTED: { + std::lock_guard lock(self->mutex_); + if (stream != self->control_stream_) break; + self->control_stream_receive_closed_ = true; + self->condition_.notify_all(); + break; + } + case QUIC_STREAM_EVENT_SHUTDOWN_COMPLETE: { + std::lock_guard lock(self->mutex_); + if (stream != self->control_stream_) break; + self->control_stream_shutdown_complete_ = true; + self->control_stream_receive_closed_ = true; + self->condition_.notify_all(); + break; + } + default: + break; + } + return QUIC_STATUS_SUCCESS; + } + + QUIC_STATUS handleControlReceive( + HQUIC stream, const QUIC_STREAM_EVENT* event) + { + if (event->RECEIVE.BufferCount != 0U && !event->RECEIVE.Buffers) { + return failControlReceive( + stream, "MsQuic supplied a null control receive buffer", + QUIC_STATUS_INVALID_PARAMETER); + } + + std::size_t received_bytes = 0U; + for (std::uint32_t index = 0U; + index < event->RECEIVE.BufferCount; ++index) { + const QUIC_BUFFER& buffer = event->RECEIVE.Buffers[index]; + if (buffer.Length != 0U && !buffer.Buffer) { + return failControlReceive( + stream, "MsQuic supplied a null control receive payload", + QUIC_STATUS_INVALID_PARAMETER); + } + if (buffer.Length > + std::numeric_limits::max() - received_bytes) { + return failControlReceive( + stream, "QUIC control receive byte count overflow", + QUIC_STATUS_BUFFER_TOO_SMALL); + } + received_bytes += buffer.Length; + } + if (received_bytes == 0U) return QUIC_STATUS_SUCCESS; + + std::size_t queue_limit = 0U; + { + std::lock_guard lock(mutex_); + if (stream != control_stream_) return QUIC_STATUS_SUCCESS; + queue_limit = maximum_control_receive_queue_bytes_; + } + if (received_bytes > queue_limit) { + return failControlReceive( + stream, "QUIC control receive chunk exceeds the bounded queue", + QUIC_STATUS_BUFFER_TOO_SMALL); + } + + std::vector chunk; + try { + chunk.resize(received_bytes); + std::size_t offset = 0U; + for (std::uint32_t index = 0U; + index < event->RECEIVE.BufferCount; ++index) { + const QUIC_BUFFER& buffer = event->RECEIVE.Buffers[index]; + std::copy_n(buffer.Buffer, buffer.Length, + chunk.begin() + static_cast(offset)); + offset += buffer.Length; + } + } catch (...) { + return failControlReceive( + stream, "unable to allocate QUIC control receive storage", + QUIC_STATUS_OUT_OF_MEMORY); + } + + { + std::lock_guard lock(mutex_); + if (stream != control_stream_ || + control_stream_receive_closed_ || !connected_) { + return QUIC_STATUS_SUCCESS; + } + if (control_receive_queue_.size() >= + kMaximumControlReceiveQueueChunks || + control_receive_queue_bytes_ > + maximum_control_receive_queue_bytes_ - received_bytes) { + control_receive_error_ = + "QUIC control receive queue capacity exceeded"; + control_stream_receive_closed_ = true; + connected_ = false; + condition_.notify_all(); + return QUIC_STATUS_BUFFER_TOO_SMALL; + } + try { + control_receive_queue_.push_back(std::move(chunk)); + } catch (...) { + control_receive_error_ = + "unable to enqueue QUIC control receive bytes"; + control_stream_receive_closed_ = true; + connected_ = false; + condition_.notify_all(); + return QUIC_STATUS_OUT_OF_MEMORY; + } + control_receive_queue_bytes_ += received_bytes; + condition_.notify_all(); + } + // Returning success consumes every buffer synchronously. Their bytes + // are now owned by control_receive_queue_. + return QUIC_STATUS_SUCCESS; + } + + QUIC_STATUS failControlReceive( + HQUIC stream, const char* message, const QUIC_STATUS status) + { + std::lock_guard lock(mutex_); + if (stream != control_stream_) return QUIC_STATUS_SUCCESS; + control_receive_error_ = message; + control_stream_receive_closed_ = true; + connected_ = false; + condition_.notify_all(); + return status; + } + + bool openApi(std::string* error) + { + if (api_) return true; + QUIC_STATUS status = MsQuicOpen2(&api_); + if (QUIC_FAILED(status)) { + setError(error, statusText("MsQuicOpen2", status)); + return false; + } + const QUIC_REGISTRATION_CONFIG registration_config = { + "cmvr-es-quic-edge", QUIC_EXECUTION_PROFILE_LOW_LATENCY}; + status = api_->RegistrationOpen(®istration_config, ®istration_); + if (QUIC_FAILED(status)) { + setError(error, statusText("RegistrationOpen", status)); + MsQuicClose(api_); + api_ = nullptr; + return false; + } + return true; + } + + bool openConfiguration(const config::QuicEdgeConfig& config, + std::string* error) + { + QUIC_BUFFER alpn{}; + alpn.Buffer = reinterpret_cast( + const_cast(config.alpn().data())); + alpn.Length = static_cast(config.alpn().size()); + QUIC_SETTINGS settings{}; + settings.IsSet.DatagramReceiveEnabled = TRUE; + settings.DatagramReceiveEnabled = TRUE; + settings.IsSet.IdleTimeoutMs = TRUE; + const std::uint64_t heartbeat_safe_idle_timeout = + kMaximumHeartbeatIntervalMs + + 2ULL * config.control_response_timeout_ms() + 60000ULL; + settings.IdleTimeoutMs = std::max( + heartbeat_safe_idle_timeout, + std::max( + 30000ULL, config.reconnect().connect_timeout_ms() * 3ULL)); + QUIC_STATUS status = api_->ConfigurationOpen( + registration_, &alpn, 1, &settings, sizeof(settings), + nullptr, &configuration_); + if (QUIC_FAILED(status)) { + setError(error, statusText("ConfigurationOpen", status)); + return false; + } + + QUIC_CREDENTIAL_CONFIG credential{}; + credential.Flags = QUIC_CREDENTIAL_FLAG_CLIENT; + if (config.tls().allow_insecure()) { + credential.Flags = static_cast( + credential.Flags | QUIC_CREDENTIAL_FLAG_NO_CERTIFICATE_VALIDATION); + } else { + credential.Flags = static_cast( + credential.Flags | QUIC_CREDENTIAL_FLAG_SET_CA_CERTIFICATE_FILE); + credential.CaCertificateFile = config.tls().ca_file().c_str(); + } + QUIC_CERTIFICATE_FILE certificate{}; + if (!config.tls().certificate_file().empty()) { + certificate.CertificateFile = config.tls().certificate_file().c_str(); + certificate.PrivateKeyFile = config.tls().private_key_file().c_str(); + credential.Type = QUIC_CREDENTIAL_TYPE_CERTIFICATE_FILE; + credential.CertificateFile = &certificate; + } else { + credential.Type = QUIC_CREDENTIAL_TYPE_NONE; + } + status = api_->ConfigurationLoadCredential(configuration_, &credential); + if (QUIC_FAILED(status)) { + setError(error, statusText("ConfigurationLoadCredential", status)); + api_->ConfigurationClose(configuration_); + configuration_ = nullptr; + return false; + } + return true; + } + + void closeConnectionHandles() + { + std::lock_guard operation_lock(api_operation_mutex_); + HQUIC stream = nullptr; + HQUIC connection = nullptr; + HQUIC configuration = nullptr; + { + std::lock_guard lock(mutex_); + stream = std::exchange(control_stream_, nullptr); + connection = std::exchange(connection_, nullptr); + configuration = std::exchange(configuration_, nullptr); + connected_ = false; + datagram_send_enabled_ = false; + maximum_datagram_bytes_ = 0U; + control_stream_start_completed_ = false; + control_stream_started_ = false; + control_stream_shutdown_complete_ = false; + control_stream_receive_closed_ = true; + ++control_receive_epoch_; + condition_.notify_all(); + } + if (!api_) return; + if (stream) api_->StreamClose(stream); + if (connection) api_->ConnectionClose(connection); + if (configuration) api_->ConfigurationClose(configuration); + std::vector abandoned_sends; + { + // ConnectionClose/StreamClose are the final handle calls. Once + // they return, no send callback may still reference these + // contexts, so any missing completion can be reclaimed safely. + std::lock_guard lock(mutex_); + abandoned_sends.assign(outstanding_sends_.begin(), + outstanding_sends_.end()); + outstanding_sends_.clear(); + pending_send_count_ = 0U; + } + for (auto* context : abandoned_sends) delete context; + } + + void finishSend(SendContext* context) + { + if (!context) return; + bool owned = false; + { + std::lock_guard lock(mutex_); + owned = outstanding_sends_.erase(context) != 0U; + if (owned && pending_send_count_ != 0U) --pending_send_count_; + condition_.notify_all(); + } + if (owned) delete context; + } + + void registerSend(SendContext* context) + { + std::lock_guard lock(mutex_); + outstanding_sends_.insert(context); + } + + void releaseReservedSends(const std::size_t count) + { + std::lock_guard lock(mutex_); + pending_send_count_ = count > pending_send_count_ + ? 0U : pending_send_count_ - count; + condition_.notify_all(); + } + + static void setError(std::string* error, const std::string& message) + { + if (error) *error = message; + } + + const std::size_t send_queue_depth_; + const std::size_t reserved_control_sends_; + const QUIC_API_TABLE* api_{nullptr}; + HQUIC registration_{nullptr}; + HQUIC configuration_{nullptr}; + HQUIC connection_{nullptr}; + HQUIC control_stream_{nullptr}; + + mutable std::mutex api_operation_mutex_; + mutable std::mutex mutex_; + std::condition_variable condition_; + bool connected_{false}; + bool shutdown_complete_{false}; + bool datagram_send_enabled_{false}; + bool control_stream_start_completed_{false}; + bool control_stream_started_{false}; + bool control_stream_shutdown_complete_{false}; + bool control_stream_receive_closed_{true}; + std::size_t maximum_datagram_bytes_{0}; + std::size_t maximum_control_receive_queue_bytes_{ + kMinimumControlReceiveQueueBytes}; + std::size_t control_receive_queue_bytes_{0}; + std::uint64_t control_receive_epoch_{0}; + std::size_t pending_send_count_{0}; + std::deque> control_receive_queue_; + std::unordered_set outstanding_sends_; + std::string control_receive_error_; + std::string connect_error_; +}; + +} // namespace + +std::unique_ptr createMsQuicTransport( + const std::size_t send_queue_depth) +{ + return std::make_unique(send_queue_depth); +} + +} // namespace cmvr::quic_edge + +#endif // CMVR_HAS_MSQUIC diff --git a/cmvr-es/service/quic_edge/src/quic_edge_device_adapter.cpp b/cmvr-es/service/quic_edge/src/quic_edge_device_adapter.cpp new file mode 100644 index 00000000..f17556a1 --- /dev/null +++ b/cmvr-es/service/quic_edge/src/quic_edge_device_adapter.cpp @@ -0,0 +1,67 @@ +#include "service/quic_edge/include/quic_edge_service.h" + +#include + +#include "manager/device_manager/include/device_manager.h" +#include "manager/media_source_hub/include/device_media_source_adapter.h" + +namespace cmvr::quic_edge { + +QuicEdgeService::QuicEdgeService(config::QuicEdgeConfig config) + : config_(std::move(config)), + transport_(createDefaultQuicTransport(config_.datagram_send_queue_depth())), + media_hub_(&media::globalMediaSourceHub()), + using_default_transport_(true), + using_global_media_hub_(true) +{ + initializeIdentity(); + source_registrar_ = [this]( + const config::QuicEdgeTrackConfig& track, + const std::string& source_track_id, + std::string* error) { + auto& manager = device::DeviceManager::getInstance(); + bool registered = false; + switch (track.source_kind()) { + case config::QuicEdgeTrackConfig::SOURCE_KIND_CAMERA: { + auto camera = + manager.getDevice(track.device_id()); + if (!camera) { + if (error) { + *error = "camera unavailable for QUIC edge: " + + track.device_id(); + } + return false; + } + registered = media::ensureCameraMediaSource(*media_hub_, camera); + break; + } + case config::QuicEdgeTrackConfig::SOURCE_KIND_MICROPHONE: { + auto microphone = manager.getDevice( + track.device_id()); + if (!microphone) { + if (error) { + *error = "microphone unavailable for QUIC edge: " + + track.device_id(); + } + return false; + } + registered = media::ensureMicrophoneMediaSource( + *media_hub_, microphone); + break; + } + case config::QuicEdgeTrackConfig::SOURCE_KIND_UNSPECIFIED: + if (error) *error = "unsupported QUIC edge source kind"; + return false; + } + if (!registered && !media_hub_->hasSource(source_track_id)) { + if (error) { + *error = "failed to register MediaSourceHub source: " + + source_track_id; + } + return false; + } + return true; + }; +} + +} // namespace cmvr::quic_edge diff --git a/cmvr-es/service/quic_edge/src/quic_edge_service.cpp b/cmvr-es/service/quic_edge/src/quic_edge_service.cpp new file mode 100644 index 00000000..53c964e0 --- /dev/null +++ b/cmvr-es/service/quic_edge/src/quic_edge_service.cpp @@ -0,0 +1,1648 @@ +#include "service/quic_edge/include/quic_edge_service.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "cmvr/quic_edge/v1/quic_edge.pb.h" + +namespace cmvr::quic_edge { +namespace { + +constexpr std::uint32_t kMinimumDatagramBytes = + static_cast(kDatagramHeaderBytes + 1U); +constexpr std::uint32_t kMaximumDatagramBytes = 65527U; +constexpr std::uint32_t kMaximumControlBytes = 16U * 1024U * 1024U; +constexpr std::uint32_t kMaximumConfiguredFrameBytes = 256U * 1024U * 1024U; +constexpr std::size_t kMaximumDrainPerPoll = 4096U; +constexpr std::uint32_t kMinimumHeartbeatIntervalMs = 250U; +constexpr std::uint32_t kMaximumHeartbeatIntervalMs = 60U * 60U * 1000U; +constexpr auto kControlPollInterval = std::chrono::milliseconds(50); +constexpr auto kMediaSourceRetryInterval = std::chrono::seconds(1); +constexpr std::size_t kMaximumReservedControlSends = 8U; + +void setError(std::string* error, const std::string& message) +{ + if (error) { + *error = message; + } +} + +std::string trim(std::string value) +{ + const auto first = value.find_first_not_of(" \t\r\n"); + if (first == std::string::npos) return {}; + const auto last = value.find_last_not_of(" \t\r\n"); + return value.substr(first, last - first + 1U); +} + +std::string readTextFile(const std::string& path) +{ + std::ifstream input(path, std::ios::in | std::ios::binary); + if (!input) return {}; + std::ostringstream contents; + contents << input.rdbuf(); + return input.good() || input.eof() ? contents.str() : std::string{}; +} + +std::string randomBootId() +{ + std::random_device random_device; + std::mt19937_64 generator(random_device()); + std::uniform_int_distribution distribution; + const std::uint64_t high = distribution(generator); + const std::uint64_t low = distribution(generator); + std::ostringstream stream; + stream << std::hex << std::setfill('0') + << std::setw(8) << static_cast(high >> 32U) << '-' + << std::setw(4) << static_cast(high >> 16U) << '-' + << std::setw(4) << static_cast(high) << '-' + << std::setw(4) << static_cast(low >> 48U) << '-' + << std::setw(12) << (low & 0x0000ffffffffffffULL); + return stream.str(); +} + +std::string resolveNodeId(const std::string& configured_node_id) +{ + if (!configured_node_id.empty() && configured_node_id != "auto") { + return configured_node_id; + } + std::array hostname{}; + if (::gethostname(hostname.data(), hostname.size() - 1U) == 0) { + hostname.back() = '\0'; + const std::string resolved = trim(hostname.data()); + if (!resolved.empty()) return resolved; + } + return "cmvr-es"; +} + +std::string resolveBootId() +{ + const std::string kernel_boot_id = + trim(readTextFile("/proc/sys/kernel/random/boot_id")); + return kernel_boot_id.empty() ? randomBootId() : kernel_boot_id; +} + +std::uint64_t unixTimeMs() +{ + const auto now = std::chrono::system_clock::now().time_since_epoch(); + return static_cast( + std::chrono::duration_cast(now).count()); +} + +std::vector collectInterfaceAddresses( + const bool include_loopback) +{ + std::vector addresses; + ifaddrs* interface_list = nullptr; + if (::getifaddrs(&interface_list) != 0 || interface_list == nullptr) { + return addresses; + } + + std::set seen; + for (const ifaddrs* current = interface_list; + current != nullptr; current = current->ifa_next) { + if (!current->ifa_addr || !current->ifa_name || + (current->ifa_flags & IFF_UP) == 0) { + continue; + } + const bool loopback = (current->ifa_flags & IFF_LOOPBACK) != 0; + if (loopback && !include_loopback) continue; + + const int family = current->ifa_addr->sa_family; + std::array buffer{}; + const void* source = nullptr; + v1::NetworkInterfaceAddress::AddressFamily proto_family = + v1::NetworkInterfaceAddress::ADDRESS_FAMILY_UNSPECIFIED; + if (family == AF_INET) { + source = &reinterpret_cast( + current->ifa_addr)->sin_addr; + proto_family = v1::NetworkInterfaceAddress::ADDRESS_FAMILY_IPV4; + } else if (family == AF_INET6) { + source = &reinterpret_cast( + current->ifa_addr)->sin6_addr; + proto_family = v1::NetworkInterfaceAddress::ADDRESS_FAMILY_IPV6; + } else { + continue; + } + if (::inet_ntop(family, source, buffer.data(), buffer.size()) == nullptr) { + continue; + } + const std::string ip_address(buffer.data()); + if (ip_address.empty() || ip_address == "0.0.0.0" || ip_address == "::") { + continue; + } + const std::string key = std::string(current->ifa_name) + '\n' + ip_address; + if (!seen.insert(key).second) continue; + + v1::NetworkInterfaceAddress address; + address.set_interface_name(current->ifa_name); + address.set_ip_address(ip_address); + address.set_family(proto_family); + address.set_loopback(loopback); + addresses.push_back(std::move(address)); + } + ::freeifaddrs(interface_list); + + std::sort(addresses.begin(), addresses.end(), + [](const v1::NetworkInterfaceAddress& lhs, + const v1::NetworkInterfaceAddress& rhs) { + if (lhs.loopback() != rhs.loopback()) return !lhs.loopback(); + if (lhs.interface_name() != rhs.interface_name()) { + return lhs.interface_name() < rhs.interface_name(); + } + if (lhs.family() != rhs.family()) return lhs.family() < rhs.family(); + return lhs.ip_address() < rhs.ip_address(); + }); + return addresses; +} + +std::string advertisedGrpcHost( + const config::QuicEdgeConfig& config, + const std::vector& interfaces) +{ + if (!config.grpc_endpoint_host().empty() && + config.grpc_endpoint_host() != "auto") { + return config.grpc_endpoint_host(); + } + for (const auto& address : interfaces) { + if (!address.loopback() && + address.family() == v1::NetworkInterfaceAddress::ADDRESS_FAMILY_IPV4) { + return address.ip_address(); + } + } + for (const auto& address : interfaces) { + if (!address.loopback()) return address.ip_address(); + } + return interfaces.empty() ? "127.0.0.1" : interfaces.front().ip_address(); +} + +void populateDescriptor(const config::QuicEdgeConfig& config, + const std::string& node_id, + const std::string& boot_id, + const std::string& software_version, + v1::NodeDescriptor* descriptor) +{ + if (!descriptor) return; + descriptor->set_node_id(node_id); + descriptor->set_boot_id(boot_id); + descriptor->set_software_version(software_version); + const auto interfaces = + collectInterfaceAddresses(config.include_loopback_interfaces()); + for (const auto& address : interfaces) { + *descriptor->add_local_interfaces() = address; + } + auto* endpoint = descriptor->mutable_grpc_endpoint(); + endpoint->set_host(advertisedGrpcHost(config, interfaces)); + endpoint->set_port(config.grpc_endpoint_port()); + endpoint->set_tls(config.grpc_endpoint_tls()); +} + +void populateHeartbeatNetwork(const config::QuicEdgeConfig& config, + v1::NodeHeartbeat* heartbeat) +{ + if (!heartbeat) return; + const auto interfaces = + collectInterfaceAddresses(config.include_loopback_interfaces()); + for (const auto& address : interfaces) { + *heartbeat->add_local_interfaces() = address; + } + auto* endpoint = heartbeat->mutable_grpc_endpoint(); + endpoint->set_host(advertisedGrpcHost(config, interfaces)); + endpoint->set_port(config.grpc_endpoint_port()); + endpoint->set_tls(config.grpc_endpoint_tls()); +} + +v1::MediaKind toProtoKind(const media::MediaKind kind) +{ + switch (kind) { + case media::MediaKind::VIDEO: return v1::MEDIA_KIND_VIDEO; + case media::MediaKind::AUDIO: return v1::MEDIA_KIND_AUDIO; + case media::MediaKind::UNKNOWN: break; + } + return v1::MEDIA_KIND_UNSPECIFIED; +} + +media::MediaKind expectedKind(const config::QuicEdgeTrackConfig& track) +{ + switch (track.source_kind()) { + case config::QuicEdgeTrackConfig::SOURCE_KIND_CAMERA: + return media::MediaKind::VIDEO; + case config::QuicEdgeTrackConfig::SOURCE_KIND_MICROPHONE: + return media::MediaKind::AUDIO; + case config::QuicEdgeTrackConfig::SOURCE_KIND_UNSPECIFIED: + break; + } + return media::MediaKind::UNKNOWN; +} + +std::string sourceTrackId(const config::QuicEdgeTrackConfig& track) +{ + if (!track.source_track_id().empty()) { + return track.source_track_id(); + } + switch (track.source_kind()) { + case config::QuicEdgeTrackConfig::SOURCE_KIND_CAMERA: + return track.device_id() + "/video/color"; + case config::QuicEdgeTrackConfig::SOURCE_KIND_MICROPHONE: + return track.device_id() + "/audio/main"; + case config::QuicEdgeTrackConfig::SOURCE_KIND_UNSPECIFIED: + break; + } + return {}; +} + +std::size_t reservedControlSendSlots(const std::size_t queue_depth) +{ + if (queue_depth <= 1U) return 0U; + return std::min( + kMaximumReservedControlSends, + std::max(1U, queue_depth / 16U)); +} + +} // namespace + +const char* toString(const QuicEdgeServiceState state) +{ + switch (state) { + case QuicEdgeServiceState::UNINITIALIZED: return "UNINITIALIZED"; + case QuicEdgeServiceState::DISABLED: return "DISABLED"; + case QuicEdgeServiceState::STOPPED: return "STOPPED"; + case QuicEdgeServiceState::CONNECTING: return "CONNECTING"; + case QuicEdgeServiceState::REGISTERING: return "REGISTERING"; + case QuicEdgeServiceState::ONLINE: return "ONLINE"; + case QuicEdgeServiceState::BACKOFF: return "BACKOFF"; + case QuicEdgeServiceState::FAILED: return "FAILED"; + } + return "UNKNOWN"; +} + +QuicEdgeService::QuicEdgeService(config::QuicEdgeConfig config, + std::unique_ptr transport, + media::MediaSourceHub& media_hub) + : config_(std::move(config)), + transport_(std::move(transport)), + media_hub_(&media_hub) +{ + initializeIdentity(); +} + +QuicEdgeService::~QuicEdgeService() +{ + stop(); +} + +bool QuicEdgeService::validateConfig(const config::QuicEdgeConfig& config, + std::string* error) +{ + if (config.id().empty()) { + setError(error, "QUIC edge task id is empty"); + return false; + } + if (!config.enable()) { + return true; + } + if (config.server_host().empty()) { + setError(error, "QUIC edge server_host is empty"); + return false; + } + if (config.server_port() == 0U || config.server_port() > 65535U) { + setError(error, "QUIC edge server_port must be in [1, 65535]"); + return false; + } + if (config.alpn().empty() || config.alpn().size() > 255U) { + setError(error, "QUIC edge ALPN must contain 1 to 255 bytes"); + return false; + } + if (!config.has_tls()) { + setError(error, "QUIC edge TLS configuration is missing"); + return false; + } + if (!config.tls().allow_insecure() && + (config.tls().ca_file().empty() || config.tls().server_name().empty())) { + setError(error, "secure QUIC edge TLS requires ca_file and server_name"); + return false; + } + if (!config.tls().allow_insecure() && + config.tls().server_name() != config.server_host()) { + setError(error, + "QUIC edge v1 requires tls.server_name to match server_host"); + return false; + } + const bool has_certificate = !config.tls().certificate_file().empty(); + const bool has_private_key = !config.tls().private_key_file().empty(); + if (has_certificate != has_private_key) { + setError(error, "QUIC edge client certificate and private key must be configured together"); + return false; + } + if (!config.has_reconnect() || config.reconnect().initial_delay_ms() == 0U || + config.reconnect().maximum_delay_ms() < config.reconnect().initial_delay_ms() || + config.reconnect().connect_timeout_ms() == 0U) { + setError(error, "QUIC edge reconnect configuration is invalid"); + return false; + } + if (!std::isfinite(config.reconnect().multiplier()) || + config.reconnect().multiplier() < 1.0 || + config.reconnect().jitter_percent() > 100U) { + setError(error, "QUIC edge reconnect multiplier or jitter is invalid"); + return false; + } + if (config.maximum_datagram_bytes() < kMinimumDatagramBytes || + config.maximum_datagram_bytes() > kMaximumDatagramBytes) { + setError(error, "QUIC edge maximum_datagram_bytes is outside the v1 range"); + return false; + } + if (config.maximum_control_frame_bytes() == 0U || + config.maximum_control_frame_bytes() > kMaximumControlBytes) { + setError(error, "QUIC edge maximum_control_frame_bytes is invalid"); + return false; + } + if (config.maximum_frame_bytes() == 0U || + config.maximum_frame_bytes() > kMaximumConfiguredFrameBytes) { + setError(error, "QUIC edge maximum_frame_bytes is invalid"); + return false; + } + const std::uint64_t fragment_payload = + config.maximum_datagram_bytes() - kDatagramHeaderBytes; + if (config.maximum_frame_bytes() > fragment_payload * 65535U) { + setError(error, "QUIC edge maximum_frame_bytes exceeds v1 fragment capacity"); + return false; + } + if (config.datagram_send_queue_depth() < 2U) { + setError(error, "QUIC edge datagram_send_queue_depth must be at least 2"); + return false; + } + const bool has_enabled_media = std::any_of( + config.tracks().begin(), config.tracks().end(), + [](const config::QuicEdgeTrackConfig& track) { return track.enable(); }); + const std::uint64_t media_send_slots = + config.datagram_send_queue_depth() - + reservedControlSendSlots(config.datagram_send_queue_depth()); + if (has_enabled_media && + config.maximum_frame_bytes() > fragment_payload * media_send_slots) { + setError(error, + "maximum_frame_bytes exceeds the atomic DATAGRAM send-queue capacity"); + return false; + } + if (config.media_poll_interval_ms() == 0U || + config.media_poll_interval_ms() > 1000U) { + setError(error, "QUIC edge media_poll_interval_ms must be in [1, 1000]"); + return false; + } + if (config.grpc_endpoint_port() == 0U || + config.grpc_endpoint_port() > 65535U) { + setError(error, "advertised gRPC endpoint port must be in [1, 65535]"); + return false; + } + if (config.heartbeat_interval_ms() < kMinimumHeartbeatIntervalMs || + config.heartbeat_interval_ms() > kMaximumHeartbeatIntervalMs) { + setError(error, "heartbeat_interval_ms must be in [250, 3600000]"); + return false; + } + if (config.control_response_timeout_ms() == 0U || + config.control_response_timeout_ms() > kMaximumHeartbeatIntervalMs) { + setError(error, "control_response_timeout_ms must be in [1, 3600000]"); + return false; + } + + std::unordered_set wire_track_ids; + std::unordered_set source_track_ids; + for (const auto& track : config.tracks()) { + if (!track.enable()) { + continue; + } + if (track.track_id() == 0U || + !wire_track_ids.insert(track.track_id()).second) { + setError(error, "enabled QUIC edge track IDs must be unique and non-zero"); + return false; + } + if (expectedKind(track) == media::MediaKind::UNKNOWN) { + setError(error, "enabled QUIC edge track has unsupported source_kind"); + return false; + } + if (track.device_id().empty()) { + setError(error, "enabled QUIC edge track has empty device_id"); + return false; + } + const std::string source_id = sourceTrackId(track); + if (source_id.empty() || !source_track_ids.insert(source_id).second) { + setError(error, "enabled QUIC edge source track IDs must be unique and non-empty"); + return false; + } + if (track.max_frame_bytes() > config.maximum_frame_bytes()) { + setError(error, "track max_frame_bytes exceeds the global frame limit"); + return false; + } + } + return true; +} + +bool QuicEdgeService::initialize(std::string* error) +{ + std::lock_guard lifecycle_lock(lifecycle_mutex_); + if (worker_.joinable()) { + setError(error, "cannot initialize QUIC edge service while it is running"); + return false; + } + std::string validation_error; + if (!validateConfig(config_, &validation_error)) { + setState(QuicEdgeServiceState::FAILED, validation_error); + setError(error, validation_error); + return false; + } + if (!config_.enable()) { + setState(QuicEdgeServiceState::DISABLED); + return true; + } + if (!transport_ || !media_hub_) { + const std::string message = "QUIC edge transport or MediaSourceHub is null"; + setState(QuicEdgeServiceState::FAILED, message); + setError(error, message); + return false; + } + if (using_default_transport_ && !hasCompiledMsQuicSupport()) { + const std::string message = + "QUIC edge is enabled but CMVR_HAS_MSQUIC is not compiled"; + setState(QuicEdgeServiceState::FAILED, message); + setError(error, message); + return false; + } + setState(QuicEdgeServiceState::STOPPED); + return true; +} + +bool QuicEdgeService::start(std::string* error) +{ + std::lock_guard lifecycle_lock(lifecycle_mutex_); + { + std::lock_guard lock(mutex_); + if (state_ == QuicEdgeServiceState::DISABLED) { + return true; + } + if (state_ == QuicEdgeServiceState::UNINITIALIZED) { + setError(error, "QUIC edge service is not initialized"); + return false; + } + if (state_ == QuicEdgeServiceState::FAILED) { + setError(error, last_error_); + return false; + } + if (worker_.joinable()) { + return true; + } + stop_requested_ = false; + } + try { + worker_ = std::thread(&QuicEdgeService::run, this); + } catch (const std::exception& exception) { + const std::string message = + std::string("failed to start QUIC edge worker: ") + exception.what(); + setState(QuicEdgeServiceState::FAILED, message); + setError(error, message); + return false; + } + return true; +} + +void QuicEdgeService::stop() +{ + std::lock_guard lifecycle_lock(lifecycle_mutex_); + { + std::lock_guard lock(mutex_); + stop_requested_ = true; + media_stop_requested_ = true; + } + stop_cv_.notify_all(); + media_stop_cv_.notify_all(); + if (transport_) { + transport_->disconnect(); + } + if (worker_.joinable()) { + worker_.join(); + } + resetConnectionStatus(); + std::lock_guard lock(mutex_); + if (state_ != QuicEdgeServiceState::UNINITIALIZED && + state_ != QuicEdgeServiceState::DISABLED && + state_ != QuicEdgeServiceState::FAILED) { + state_ = QuicEdgeServiceState::STOPPED; + } +} + +QuicEdgeServiceState QuicEdgeService::state() const +{ + std::lock_guard lock(mutex_); + return state_; +} + +std::string QuicEdgeService::lastError() const +{ + std::lock_guard lock(mutex_); + return last_error_; +} + +QuicEdgeStats QuicEdgeService::stats() const +{ + std::lock_guard lock(mutex_); + return stats_; +} + +QuicEdgeStatus QuicEdgeService::status() const +{ + std::lock_guard lock(mutex_); + QuicEdgeStatus result; + result.registered = registered_; + result.node_id = node_id_; + result.boot_id = boot_id_; + result.session_id = session_id_; + result.observed_source_ip = observed_source_ip_; + result.last_media_error = last_media_error_; + result.heartbeat_sequence = heartbeat_sequence_; + result.last_heartbeat_ack_unix_ms = last_heartbeat_ack_unix_ms_; + result.active_media_tracks = active_media_tracks_; + return result; +} + +void QuicEdgeService::run() +{ + try { + auto backoff = std::chrono::milliseconds(config_.reconnect().initial_delay_ms()); + while (true) { + { + std::lock_guard lock(mutex_); + if (stop_requested_) break; + ++stats_.connection_attempts; + } + resetConnectionStatus(); + setState(QuicEdgeServiceState::CONNECTING); + + std::string error; + if (!transport_->connect( + config_, + std::chrono::milliseconds(config_.reconnect().connect_timeout_ms()), + &error)) { + setState(QuicEdgeServiceState::BACKOFF, + error.empty() ? "QUIC connect failed" : error); + if (waitForStop(jittered(backoff))) break; + backoff = nextBackoff(backoff); + continue; + } + + std::uint64_t acknowledged_heartbeats_at_connect = 0U; + { + std::lock_guard lock(mutex_); + ++stats_.successful_connections; + if (stats_.successful_connections > 1U) ++stats_.reconnects; + acknowledged_heartbeats_at_connect = + stats_.heartbeats_acknowledged; + } + session_epoch_ = nextSessionEpoch(); + control_message_sequence_ = 0U; + inbound_message_sequence_ = 0U; + has_inbound_message_sequence_ = false; + ControlFrameDecoder decoder(config_.maximum_control_frame_bytes()); + setState(QuicEdgeServiceState::REGISTERING); + + if (!performRegistration(&decoder, &error)) { + transport_->disconnect(); + { + std::lock_guard lock(mutex_); + if (stop_requested_) break; + } + setState(QuicEdgeServiceState::BACKOFF, error); + if (waitForStop(jittered(backoff))) break; + backoff = nextBackoff(backoff); + continue; + } + setState(QuicEdgeServiceState::ONLINE); + + if (!startMediaWorker(&error)) { + transport_->disconnect(); + { + std::lock_guard lock(mutex_); + if (stop_requested_) break; + } + setState(QuicEdgeServiceState::BACKOFF, error); + if (waitForStop(jittered(backoff))) break; + backoff = nextBackoff(backoff); + continue; + } + + bool connection_failed = false; + bool connection_became_healthy = false; + while (true) { + { + std::lock_guard lock(mutex_); + if (stop_requested_) break; + } + + bool received_control = false; + if (!receiveAndDispatchControl( + &decoder, std::chrono::milliseconds(0), + &received_control, &error)) { + connection_failed = true; + break; + } + if (!transport_->isConnected()) { + // receiveControl drains bytes already delivered by MsQuic + // before it reports the disconnect on the next iteration. + continue; + } + if (!connection_became_healthy) { + std::lock_guard lock(mutex_); + if (stats_.heartbeats_acknowledged > + acknowledged_heartbeats_at_connect) { + backoff = std::chrono::milliseconds( + config_.reconnect().initial_delay_ms()); + connection_became_healthy = true; + } + } + + const auto now = std::chrono::steady_clock::now(); + std::uint64_t outstanding_sequence = 0U; + std::chrono::steady_clock::time_point heartbeat_deadline; + { + std::lock_guard lock(mutex_); + outstanding_sequence = outstanding_heartbeat_sequence_; + heartbeat_deadline = heartbeat_deadline_; + } + if (outstanding_sequence != 0U && now >= heartbeat_deadline) { + { + std::lock_guard lock(mutex_); + ++stats_.heartbeat_timeouts; + } + error = "QUIC node heartbeat acknowledgement timed out"; + connection_failed = true; + break; + } + + if (outstanding_sequence == 0U && now >= next_heartbeat_) { + std::uint64_t sequence = 0U; + { + std::lock_guard lock(mutex_); + if (heartbeat_sequence_ == + std::numeric_limits::max()) { + heartbeat_sequence_ = 0U; + } + sequence = ++heartbeat_sequence_; + } + if (!sendHeartbeat(sequence, &error)) { + connection_failed = true; + break; + } + { + std::lock_guard lock(mutex_); + outstanding_heartbeat_sequence_ = sequence; + heartbeat_deadline_ = now + std::chrono::milliseconds( + config_.control_response_timeout_ms()); + ++stats_.heartbeats_sent; + } + next_heartbeat_ = now + std::chrono::milliseconds( + effective_heartbeat_interval_ms_); + } + + { + std::lock_guard lock(mutex_); + if (media_connection_failed_) { + error = media_connection_error_.empty() + ? "QUIC media worker reported a connection failure" + : media_connection_error_; + connection_failed = true; + } + } + if (connection_failed) break; + + if (!receiveAndDispatchControl( + &decoder, kControlPollInterval, + &received_control, &error)) { + connection_failed = true; + break; + } + } + + stopMediaWorker(); + transport_->disconnect(); + resetConnectionStatus(); + { + std::lock_guard lock(mutex_); + if (stop_requested_) break; + } + setState(QuicEdgeServiceState::BACKOFF, + error.empty() ? "QUIC edge connection closed" : error); + if (waitForStop(jittered(backoff))) break; + backoff = nextBackoff(backoff); + } + stopMediaWorker(); + transport_->disconnect(); + resetConnectionStatus(); + setState(QuicEdgeServiceState::STOPPED); + } catch (const std::exception& exception) { + stopMediaWorker(); + transport_->disconnect(); + resetConnectionStatus(); + setState(QuicEdgeServiceState::FAILED, + std::string("QUIC edge worker exception: ") + exception.what()); + } catch (...) { + stopMediaWorker(); + transport_->disconnect(); + resetConnectionStatus(); + setState(QuicEdgeServiceState::FAILED, "unknown QUIC edge worker exception"); + } +} + +bool QuicEdgeService::startMediaWorker(std::string* error) +{ + if (!hasEnabledMediaTracks()) { + std::lock_guard lock(mutex_); + active_media_tracks_ = 0U; + return true; + } + if (media_worker_.joinable()) { + setError(error, "QUIC media worker is already running"); + return false; + } + { + std::lock_guard lock(mutex_); + if (stop_requested_) { + setError(error, "QUIC edge service is stopping"); + return false; + } + media_stop_requested_ = false; + media_connection_failed_ = false; + media_connection_error_.clear(); + active_media_tracks_ = 0U; + } + try { + media_worker_ = std::thread(&QuicEdgeService::runMedia, this); + } catch (const std::exception& exception) { + const std::string message = + std::string("failed to start QUIC media worker: ") + exception.what(); + recordMediaConnectionFailure(message); + setError(error, message); + return false; + } + return true; +} + +void QuicEdgeService::stopMediaWorker() +{ + { + std::lock_guard lock(mutex_); + media_stop_requested_ = true; + } + media_stop_cv_.notify_all(); + if (media_worker_.joinable()) media_worker_.join(); + std::lock_guard lock(mutex_); + active_media_tracks_ = 0U; +} + +void QuicEdgeService::recordMediaConnectionFailure(const std::string& error) +{ + { + std::lock_guard lock(mutex_); + if (!media_connection_failed_) { + media_connection_error_ = error.empty() + ? "unknown QUIC media connection failure" : error; + } + media_connection_failed_ = true; + media_stop_requested_ = true; + last_media_error_ = media_connection_error_; + } + media_stop_cv_.notify_all(); +} + +void QuicEdgeService::runMedia() +{ + std::vector tracks; + try { + next_media_source_retry_ = std::chrono::steady_clock::now(); + while (true) { + { + std::lock_guard lock(mutex_); + if (stop_requested_ || media_stop_requested_) break; + } + if (!transport_->isConnected()) break; + + const auto now = std::chrono::steady_clock::now(); + std::string error; + if (now >= next_media_source_retry_) { + tracks.erase( + std::remove_if(tracks.begin(), tracks.end(), + [](const ActiveTrack& track) { + return !track.subscription.valid(); + }), + tracks.end()); + if (!openMediaSession(&tracks, &error)) { + recordMediaConnectionFailure(error); + break; + } + next_media_source_retry_ = now + kMediaSourceRetryInterval; + } + + bool sent_anything = false; + bool failed = false; + for (auto& track : tracks) { + if (!processTrack(&track, &sent_anything, &error)) { + failed = true; + break; + } + } + if (failed) { + recordMediaConnectionFailure(error); + break; + } + tracks.erase( + std::remove_if(tracks.begin(), tracks.end(), + [](const ActiveTrack& track) { + return !track.subscription.valid(); + }), + tracks.end()); + { + std::lock_guard lock(mutex_); + active_media_tracks_ = tracks.size(); + } + + if (!sent_anything) { + const auto poll_wait = tracks.empty() + ? kControlPollInterval + : std::chrono::milliseconds(config_.media_poll_interval_ms()); + std::unique_lock lock(mutex_); + media_stop_cv_.wait_for(lock, poll_wait, [this]() { + return stop_requested_ || media_stop_requested_; + }); + } + } + } catch (const std::exception& exception) { + recordMediaConnectionFailure( + std::string("QUIC media worker exception: ") + exception.what()); + } catch (...) { + recordMediaConnectionFailure("unknown QUIC media worker exception"); + } + tracks.clear(); + std::lock_guard lock(mutex_); + active_media_tracks_ = 0U; +} + +bool QuicEdgeService::performRegistration(ControlFrameDecoder* decoder, + std::string* error) +{ + if (!decoder) { + setError(error, "control frame decoder is null"); + return false; + } + if (!sendNodeRegistration(error)) return false; + + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(config_.control_response_timeout_ms()); + while (transport_->isConnected()) { + { + std::lock_guard lock(mutex_); + if (stop_requested_) { + setError(error, "QUIC edge service is stopping"); + return false; + } + if (registered_) return true; + } + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + setError(error, "QUIC node registration response timed out"); + return false; + } + const auto remaining = std::chrono::duration_cast( + deadline - now); + bool received = false; + if (!receiveAndDispatchControl( + decoder, std::min(remaining, kControlPollInterval), + &received, error)) { + return false; + } + } + setError(error, "QUIC connection closed during node registration"); + return false; +} + +bool QuicEdgeService::sendNodeRegistration(std::string* error) +{ + std::lock_guard control_lock(control_send_mutex_); + v1::EdgeControlEnvelope envelope; + envelope.set_protocol_version(kProtocolVersion); + envelope.set_message_sequence(control_message_sequence_++); + auto* request = envelope.mutable_node_register_request(); + populateDescriptor(config_, node_id_, boot_id_, software_version_, + request->mutable_node()); + request->set_sent_at_unix_ms(unixTimeMs()); + std::string serialized; + if (!envelope.SerializeToString(&serialized)) { + setError(error, "failed to serialize NodeRegisterRequest"); + return false; + } + if (!sendControlEnvelope(serialized, error)) return false; + std::lock_guard lock(mutex_); + ++stats_.registrations_sent; + return true; +} + +bool QuicEdgeService::sendHeartbeat(const std::uint64_t sequence, + std::string* error) +{ + std::lock_guard control_lock(control_send_mutex_); + std::string session_id; + { + std::lock_guard lock(mutex_); + session_id = session_id_; + } + if (session_id.empty()) { + setError(error, "cannot send heartbeat before node registration"); + return false; + } + + v1::EdgeControlEnvelope envelope; + envelope.set_protocol_version(kProtocolVersion); + envelope.set_message_sequence(control_message_sequence_++); + auto* heartbeat = envelope.mutable_node_heartbeat(); + heartbeat->set_node_id(node_id_); + heartbeat->set_boot_id(boot_id_); + heartbeat->set_session_id(session_id); + heartbeat->set_sequence(sequence); + heartbeat->set_sent_at_unix_ms(unixTimeMs()); + heartbeat->set_software_version(software_version_); + populateHeartbeatNetwork(config_, heartbeat); + std::string serialized; + if (!envelope.SerializeToString(&serialized)) { + setError(error, "failed to serialize NodeHeartbeat"); + return false; + } + return sendControlEnvelope(serialized, error); +} + +bool QuicEdgeService::receiveAndDispatchControl( + ControlFrameDecoder* decoder, + const std::chrono::milliseconds timeout, + bool* received, + std::string* error) +{ + if (!decoder || !received) { + setError(error, "control receive arguments are null"); + return false; + } + *received = false; + std::vector chunk; + const auto result = transport_->receiveControl(&chunk, timeout, error); + if (result == TransportReceiveResult::TIMEOUT) return true; + if (result == TransportReceiveResult::DISCONNECTED) { + if (error && error->empty()) *error = "QUIC control stream disconnected"; + return false; + } + if (result == TransportReceiveResult::ERROR) { + if (error && error->empty()) *error = "QUIC control stream receive failed"; + return false; + } + if (chunk.empty()) { + setError(error, "QUIC control stream returned an empty data chunk"); + return false; + } + + std::vector> frames; + if (!decoder->push(chunk, &frames, error)) return false; + *received = true; + for (const auto& frame : frames) { + if (!dispatchControlFrame(frame, error)) return false; + } + return true; +} + +bool QuicEdgeService::dispatchControlFrame( + const std::vector& frame, + std::string* error) +{ + v1::EdgeControlEnvelope envelope; + if (!envelope.ParseFromArray(frame.data(), static_cast(frame.size()))) { + setError(error, "failed to parse EdgeControlEnvelope"); + return false; + } + if (envelope.protocol_version() != kProtocolVersion) { + setError(error, "unsupported QUIC edge protocol version"); + return false; + } + if (has_inbound_message_sequence_ && + envelope.message_sequence() <= inbound_message_sequence_) { + setError(error, "QUIC control message sequence did not increase"); + return false; + } + inbound_message_sequence_ = envelope.message_sequence(); + has_inbound_message_sequence_ = true; + + if (envelope.has_node_register_response()) { + const auto& response = envelope.node_register_response(); + if (!response.accepted()) { + { + std::lock_guard lock(mutex_); + ++stats_.registrations_rejected; + } + setError(error, response.message().empty() + ? "gateway rejected node registration" + : response.message()); + return false; + } + if (response.session_id().empty()) { + setError(error, "gateway accepted registration without a session ID"); + return false; + } + const std::uint32_t requested_interval = + response.heartbeat_interval_ms() == 0U + ? config_.heartbeat_interval_ms() + : response.heartbeat_interval_ms(); + { + std::lock_guard lock(mutex_); + if (registered_) { + setError(error, "duplicate node registration response"); + return false; + } + registered_ = true; + session_id_ = response.session_id(); + observed_source_ip_ = response.observed_source_ip(); + effective_heartbeat_interval_ms_ = std::clamp( + requested_interval, kMinimumHeartbeatIntervalMs, + kMaximumHeartbeatIntervalMs); + next_heartbeat_ = std::chrono::steady_clock::now(); + ++stats_.registrations_accepted; + } + return true; + } + + if (envelope.has_node_heartbeat_ack()) { + const auto& ack = envelope.node_heartbeat_ack(); + std::lock_guard lock(mutex_); + if (!registered_ || outstanding_heartbeat_sequence_ == 0U) { + setError(error, "unexpected heartbeat acknowledgement"); + return false; + } + if (!ack.accepted()) { + setError(error, ack.message().empty() + ? "gateway rejected node heartbeat" + : ack.message()); + return false; + } + if (ack.session_id() != session_id_ || + ack.acknowledged_sequence() != outstanding_heartbeat_sequence_) { + setError(error, "heartbeat acknowledgement session or sequence mismatch"); + return false; + } + outstanding_heartbeat_sequence_ = 0U; + if (!ack.observed_source_ip().empty()) { + observed_source_ip_ = ack.observed_source_ip(); + } + last_heartbeat_ack_unix_ms_ = unixTimeMs(); + ++stats_.heartbeats_acknowledged; + return true; + } + + if (envelope.has_protocol_error()) { + const auto& protocol_error = envelope.protocol_error(); + { + std::lock_guard lock(mutex_); + ++stats_.protocol_errors; + last_error_ = protocol_error.message(); + } + if (protocol_error.fatal()) { + setError(error, protocol_error.message().empty() + ? "gateway reported a fatal protocol error" + : protocol_error.message()); + return false; + } + return true; + } + + setError(error, "gateway sent an unexpected QUIC control message"); + return false; +} + +bool QuicEdgeService::openMediaSession(std::vector* tracks, + std::string* error) +{ + if (!tracks) { + setError(error, "active track output is null"); + return false; + } + if (!hasEnabledMediaTracks()) { + std::lock_guard lock(mutex_); + active_media_tracks_ = 0U; + return true; + } + if (!media_session_announced_) { + if (!sendSessionOpen(error)) { + if (!transport_->isConnected()) return false; + recordMediaError(error ? *error : "failed to open media session"); + if (error) error->clear(); + return true; + } + media_session_announced_ = true; + { + std::lock_guard lock(mutex_); + ++stats_.media_sessions_opened; + } + } + refreshMediaTracks(tracks); + return true; +} + +void QuicEdgeService::refreshMediaTracks(std::vector* tracks) +{ + if (!tracks) return; + for (const auto& track_config : config_.tracks()) { + if (!track_config.enable()) continue; + const bool already_active = std::any_of( + tracks->begin(), tracks->end(), + [&track_config](const ActiveTrack& active) { + return active.config.track_id() == track_config.track_id(); + }); + if (already_active) continue; + + ActiveTrack track; + track.config = track_config; + track.source_track_id = sourceTrackId(track_config); + std::string source_error; + if (!ensureSourceRegistered(track_config, track.source_track_id, + &source_error)) { + recordMediaError(source_error); + continue; + } + track.subscription = media_hub_->subscribe( + track.source_track_id, + media::MediaSourceHub::StartPosition::LATEST_AVAILABLE, + [this] { + std::lock_guard lock(mutex_); + return stop_requested_ || media_stop_requested_; + }); + if (!track.subscription.valid()) { + recordMediaError( + "MediaSourceHub source unavailable: " + track.source_track_id); + continue; + } + track.waiting_for_keyframe = + expectedKind(track_config) == media::MediaKind::VIDEO; + track.next_frame_discontinuous = true; + if (track.waiting_for_keyframe) requestKeyFrame(&track); + tracks->push_back(std::move(track)); + } + std::lock_guard lock(mutex_); + active_media_tracks_ = tracks->size(); +} + +bool QuicEdgeService::hasEnabledMediaTracks() const +{ + return std::any_of(config_.tracks().begin(), config_.tracks().end(), + [](const config::QuicEdgeTrackConfig& track) { + return track.enable(); + }); +} + +bool QuicEdgeService::ensureSourceRegistered( + const config::QuicEdgeTrackConfig& track, + const std::string& source_track_id, + std::string* error) +{ + if (media_hub_->hasSource(source_track_id)) return true; + if (!using_global_media_hub_ || !source_registrar_) { + setError(error, "MediaSourceHub source unavailable: " + source_track_id); + return false; + } + const bool registered = source_registrar_(track, source_track_id, error); + if (!registered && !media_hub_->hasSource(source_track_id)) { + if (!error || error->empty()) { + setError(error, + "failed to register MediaSourceHub source: " + source_track_id); + } + return false; + } + if (!media_hub_->hasSource(source_track_id)) { + setError(error, + "configured source_track_id does not match the device adapter track: " + + source_track_id); + return false; + } + return true; +} + +bool QuicEdgeService::sendSessionOpen(std::string* error) +{ + std::lock_guard control_lock(control_send_mutex_); + std::string session_id; + { + std::lock_guard lock(mutex_); + session_id = session_id_; + } + if (session_id.empty()) { + setError(error, "cannot open media before node registration"); + return false; + } + v1::EdgeControlEnvelope envelope; + envelope.set_protocol_version(kProtocolVersion); + envelope.set_message_sequence(control_message_sequence_++); + auto* open = envelope.mutable_media_session_open(); + open->set_node_id(node_id_); + open->set_session_epoch(session_epoch_); + open->set_session_id(session_id); + std::string serialized; + if (!envelope.SerializeToString(&serialized)) { + setError(error, "failed to serialize MediaSessionOpen"); + return false; + } + return sendControlEnvelope(serialized, error); +} + +bool QuicEdgeService::sendTrackDescription( + const MediaTrackDescription& description, + std::string* error) +{ + std::lock_guard control_lock(control_send_mutex_); + v1::EdgeControlEnvelope envelope; + envelope.set_protocol_version(kProtocolVersion); + envelope.set_message_sequence(control_message_sequence_++); + auto* descriptor = envelope.mutable_media_track_descriptor(); + descriptor->set_track_id(description.track_id); + descriptor->set_kind(toProtoKind(description.kind)); + descriptor->set_device_id(description.source_id); + descriptor->set_codec(codecName(description.codec)); + descriptor->set_codec_generation(description.codec_generation); + descriptor->set_source_track_id(description.source_track_id); + descriptor->set_codec_generation_token(description.codec_generation_token); + descriptor->set_payload_format(payloadFormatName(description.payload_format)); + descriptor->set_width(description.width); + descriptor->set_height(description.height); + descriptor->set_frames_per_second(description.nominal_rate); + descriptor->set_sample_rate(description.sample_rate); + descriptor->set_channels(description.channels); + if (!description.codec_config.empty()) { + descriptor->set_codec_config(description.codec_config.data(), + description.codec_config.size()); + } + std::string serialized; + if (!envelope.SerializeToString(&serialized)) { + setError(error, "failed to serialize MediaTrackDescriptor"); + return false; + } + return sendControlEnvelope(serialized, error); +} + +bool QuicEdgeService::sendControlEnvelope(const std::string& serialized, + std::string* error) +{ + std::vector framed; + if (!ControlFrameEncoder::encode( + reinterpret_cast(serialized.data()), + serialized.size(), config_.maximum_control_frame_bytes(), + &framed, error)) { + return false; + } + const auto result = transport_->sendControl(std::move(framed), error); + if (result == TransportSendResult::QUEUED) return true; + if (error && error->empty()) { + *error = result == TransportSendResult::WOULD_BLOCK + ? "QUIC reliable control stream is congested" + : "QUIC reliable control stream is disconnected"; + } + return false; +} + +bool QuicEdgeService::processTrack(ActiveTrack* track, + bool* sent_anything, + std::string* error) +{ + if (!track || !sent_anything) { + setError(error, "active track or sent flag is null"); + return false; + } + auto read = track->subscription.tryRead(); + if (!read) { + if (!track->subscription.valid()) { + recordMediaError( + "MediaSourceHub subscription stopped: " + track->source_track_id); + } + return true; + } + *sent_anything = true; + const media::MediaFramePtr& frame = read->value; + if (!frame || !frame->descriptor || + frame->descriptor->id != track->source_track_id || + frame->descriptor->kind != expectedKind(track->config)) { + recordMediaError("MediaSourceHub returned an invalid or mismatched frame"); + track->next_frame_discontinuous = true; + return true; + } + + bool discontinuity = track->next_frame_discontinuous || + frame->discontinuity || + read->generation_changed || + read->dropped_since_last_read != 0U; + if (read->dropped_since_last_read != 0U) { + std::lock_guard lock(mutex_); + stats_.frames_dropped_source += read->dropped_since_last_read; + } + + const auto description = describeTrack(track->config.track_id(), + *frame->descriptor); + const bool descriptor_changed = + !track->last_description || *track->last_description != description; + if (track->last_description && descriptor_changed && + description.codec_generation < track->last_description->codec_generation) { + recordMediaError("MediaSourceHub descriptor generation regressed"); + track->next_frame_discontinuous = true; + return true; + } + if (descriptor_changed) { + if (!sendTrackDescription(description, error)) { + if (!transport_->isConnected()) return false; + recordMediaError(error ? *error : "failed to send media track descriptor"); + if (error) error->clear(); + track->next_frame_discontinuous = true; + return true; + } + track->last_description = description; + discontinuity = true; + if (description.kind == media::MediaKind::VIDEO) { + track->waiting_for_keyframe = true; + track->keyframe_requested = false; + requestKeyFrame(track); + } + } + + if (description.kind == media::MediaKind::VIDEO && discontinuity) { + if (!track->waiting_for_keyframe) { + track->keyframe_requested = false; + } + track->waiting_for_keyframe = true; + requestKeyFrame(track); + } + + if (frame->descriptor->kind == media::MediaKind::VIDEO && + track->waiting_for_keyframe && !frame->key_frame) { + track->next_frame_discontinuous = discontinuity; + requestKeyFrame(track); + std::lock_guard lock(mutex_); + ++stats_.frames_skipped_waiting_keyframe; + return true; + } + + if (frame->empty() || frame->size() > maximumFrameBytes(*track)) { + const auto drained = shedBufferedFrames(track); + track->next_frame_discontinuous = true; + if (frame->descriptor->kind == media::MediaKind::VIDEO) { + track->waiting_for_keyframe = true; + track->keyframe_requested = false; + requestKeyFrame(track); + } + std::lock_guard lock(mutex_); + stats_.frames_dropped_oversize += 1U + drained; + return true; + } + + const std::size_t peer_maximum = transport_->maximumDatagramBytes(); + if (peer_maximum <= kDatagramHeaderBytes) { + const auto drained = shedBufferedFrames(track); + track->next_frame_discontinuous = true; + if (frame->descriptor->kind == media::MediaKind::VIDEO) { + track->waiting_for_keyframe = true; + track->keyframe_requested = false; + requestKeyFrame(track); + } + std::lock_guard lock(mutex_); + stats_.frames_dropped_no_datagram += 1U + drained; + last_media_error_ = "peer/path has no usable QUIC DATAGRAM support"; + return true; + } + const std::size_t effective_maximum = + std::min(config_.maximum_datagram_bytes(), peer_maximum); + const std::size_t fragment_payload_capacity = + effective_maximum - kDatagramHeaderBytes; + const std::size_t required_fragments = + (frame->size() + fragment_payload_capacity - 1U) / + fragment_payload_capacity; + const std::size_t maximum_batch_packets = + transport_->maximumDatagramBatchPackets(); + if (maximum_batch_packets == 0U || + required_fragments > maximum_batch_packets) { + const auto drained = shedBufferedFrames(track); + track->next_frame_discontinuous = true; + if (frame->descriptor->kind == media::MediaKind::VIDEO) { + track->waiting_for_keyframe = true; + track->keyframe_requested = false; + requestKeyFrame(track); + } + std::lock_guard lock(mutex_); + stats_.frames_dropped_oversize += 1U + drained; + last_media_error_ = + "media frame exceeds the atomic DATAGRAM batch capacity"; + return true; + } + auto& next_wire_sequence = + next_wire_frame_sequence_[track->config.track_id()]; + if (next_wire_sequence == 0U) next_wire_sequence = 1U; + if (next_wire_sequence == std::numeric_limits::max()) { + setError(error, "QUIC media wire frame sequence exhausted"); + return false; + } + const std::uint64_t wire_frame_sequence = next_wire_sequence++; + std::vector packets; + if (!packetizer_.packetize( + *frame, track->config.track_id(), + description.codec_generation_token, session_epoch_, + wire_frame_sequence, discontinuity, effective_maximum, + &packets, error)) { + const std::string packet_error = + error && !error->empty() ? *error : "media frame packetization failed"; + track->next_frame_discontinuous = true; + if (frame->descriptor->kind == media::MediaKind::VIDEO) { + track->waiting_for_keyframe = true; + track->keyframe_requested = false; + requestKeyFrame(track); + } + std::lock_guard lock(mutex_); + ++stats_.frames_dropped_oversize; + last_media_error_ = packet_error; + if (error) error->clear(); + return true; + } + + const std::size_t datagram_count = packets.size(); + const auto result = transport_->sendDatagramBatch(std::move(packets), error); + if (result == TransportSendResult::QUEUED) { + track->next_frame_discontinuous = false; + if (frame->descriptor->kind == media::MediaKind::VIDEO && frame->key_frame) { + track->waiting_for_keyframe = false; + track->keyframe_requested = false; + } + std::lock_guard lock(mutex_); + ++stats_.frames_queued; + stats_.datagrams_queued += datagram_count; + return true; + } + if (result == TransportSendResult::WOULD_BLOCK) { + const auto drained = shedBufferedFrames(track); + track->next_frame_discontinuous = true; + if (frame->descriptor->kind == media::MediaKind::VIDEO) { + track->waiting_for_keyframe = true; + track->keyframe_requested = false; + requestKeyFrame(track); + } + std::lock_guard lock(mutex_); + stats_.frames_dropped_backpressure += 1U + drained; + return true; + } + if (error && error->empty()) { + *error = result == TransportSendResult::DISCONNECTED + ? "QUIC transport disconnected while sending media" + : "QUIC transport failed while sending media"; + } + if (result == TransportSendResult::DISCONNECTED || + !transport_->isConnected()) { + return false; + } + recordMediaError(error ? *error : "QUIC DATAGRAM send failed"); + if (error) error->clear(); + track->next_frame_discontinuous = true; + return true; +} + +std::uint64_t QuicEdgeService::shedBufferedFrames(ActiveTrack* track) +{ + if (!track) return 0U; + std::uint64_t drained = 0U; + while (drained < kMaximumDrainPerPoll && track->subscription.tryRead()) { + ++drained; + } + return drained; +} + +void QuicEdgeService::requestKeyFrame(ActiveTrack* track) +{ + if (!track || track->keyframe_requested || !media_hub_ || + expectedKind(track->config) != media::MediaKind::VIDEO) { + return; + } + (void)media_hub_->requestKeyFrame(track->source_track_id); + track->keyframe_requested = true; +} + +std::uint32_t QuicEdgeService::maximumFrameBytes(const ActiveTrack& track) const +{ + return track.config.max_frame_bytes() == 0U + ? config_.maximum_frame_bytes() + : track.config.max_frame_bytes(); +} + +void QuicEdgeService::initializeIdentity() +{ + node_id_ = resolveNodeId(config_.node_id()); + boot_id_ = resolveBootId(); + software_version_ = config_.software_version().empty() + ? "unknown" : config_.software_version(); + effective_heartbeat_interval_ms_ = config_.heartbeat_interval_ms(); +} + +void QuicEdgeService::resetConnectionStatus() +{ + std::lock_guard lock(mutex_); + registered_ = false; + session_id_.clear(); + observed_source_ip_.clear(); + outstanding_heartbeat_sequence_ = 0U; + active_media_tracks_ = 0U; + last_media_error_.clear(); + media_stop_requested_ = false; + media_connection_failed_ = false; + media_connection_error_.clear(); + effective_heartbeat_interval_ms_ = config_.heartbeat_interval_ms(); + next_wire_frame_sequence_.clear(); + media_session_announced_ = false; + heartbeat_deadline_ = {}; + next_heartbeat_ = {}; +} + +void QuicEdgeService::recordMediaError(const std::string& error) +{ + std::lock_guard lock(mutex_); + ++stats_.source_errors; + last_media_error_ = error.empty() ? "unknown media source error" : error; +} + +void QuicEdgeService::setState(const QuicEdgeServiceState state, + const std::string& error) +{ + std::lock_guard lock(mutex_); + state_ = state; + if (!error.empty()) { + last_error_ = error; + } else if (state == QuicEdgeServiceState::ONLINE || + state == QuicEdgeServiceState::STOPPED || + state == QuicEdgeServiceState::DISABLED) { + last_error_.clear(); + } +} + +bool QuicEdgeService::waitForStop(const std::chrono::milliseconds duration) +{ + std::unique_lock lock(mutex_); + return stop_cv_.wait_for(lock, duration, [this]() { return stop_requested_; }); +} + +std::chrono::milliseconds QuicEdgeService::nextBackoff( + const std::chrono::milliseconds current) +{ + const double multiplied = static_cast(current.count()) * + config_.reconnect().multiplier(); + const auto bounded = std::min( + multiplied, static_cast(config_.reconnect().maximum_delay_ms())); + return std::chrono::milliseconds(static_cast(bounded)); +} + +std::chrono::milliseconds QuicEdgeService::jittered( + const std::chrono::milliseconds base) +{ + const auto jitter_percent = config_.reconnect().jitter_percent(); + if (jitter_percent == 0U) return base; + thread_local std::mt19937 generator(std::random_device{}()); + const double fraction = static_cast(jitter_percent) / 100.0; + std::uniform_real_distribution distribution(1.0 - fraction, + 1.0 + fraction); + const double value = static_cast(base.count()) * distribution(generator); + return std::chrono::milliseconds( + std::max(1, static_cast(value))); +} + +std::uint64_t QuicEdgeService::nextSessionEpoch() +{ + if (session_epoch_ == 0U) { + const auto wall_clock = static_cast( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); + std::random_device random; + session_epoch_ = wall_clock ^ + (static_cast(random()) << 32U) ^ random(); + if (session_epoch_ == 0U) session_epoch_ = 1U; + return session_epoch_; + } + session_epoch_ = session_epoch_ == std::numeric_limits::max() + ? 1U : session_epoch_ + 1U; + return session_epoch_; +} + +} // namespace cmvr::quic_edge diff --git a/cmvr-es/service/quic_edge/src/quic_edge_types.cpp b/cmvr-es/service/quic_edge/src/quic_edge_types.cpp new file mode 100644 index 00000000..f12303da --- /dev/null +++ b/cmvr-es/service/quic_edge/src/quic_edge_types.cpp @@ -0,0 +1,40 @@ +#include "service/quic_edge/include/quic_edge_types.h" + +namespace cmvr::quic_edge { + +std::uint32_t descriptorGenerationToken(const std::uint64_t generation) +{ + // Fold both halves because the shared device adapter embeds its stream + // epoch in the upper 32 bits and its codec generation in the lower half. + std::uint32_t token = static_cast(generation) ^ + static_cast(generation >> 32U); + return token == 0U ? 1U : token; +} + +const char* codecName(const media::Codec codec) +{ + switch (codec) { + case media::Codec::H264: return "h264"; + case media::Codec::H265: return "h265"; + case media::Codec::OPUS: return "opus"; + case media::Codec::PCM_S16LE: return "pcm_s16le"; + case media::Codec::AAC: return "aac"; + case media::Codec::UNKNOWN: return "unknown"; + } + return "unknown"; +} + +const char* payloadFormatName(const media::PayloadFormat format) +{ + switch (format) { + case media::PayloadFormat::ANNEX_B: return "annex_b"; + case media::PayloadFormat::AVCC: return "avcc"; + case media::PayloadFormat::RAW: return "raw"; + case media::PayloadFormat::OPUS_PACKET: return "opus_packet"; + case media::PayloadFormat::AAC_ADTS: return "aac_adts"; + case media::PayloadFormat::UNKNOWN: return "unknown"; + } + return "unknown"; +} + +} // namespace cmvr::quic_edge diff --git a/cmvr-es/service/quic_edge/src/quic_transport.cpp b/cmvr-es/service/quic_edge/src/quic_transport.cpp new file mode 100644 index 00000000..6791adbc --- /dev/null +++ b/cmvr-es/service/quic_edge/src/quic_transport.cpp @@ -0,0 +1,103 @@ +#include "service/quic_edge/include/quic_transport.h" + +#include + +namespace cmvr::quic_edge { +#ifdef CMVR_HAS_MSQUIC +std::unique_ptr createMsQuicTransport( + std::size_t send_queue_depth); +#endif + +TransportReceiveResult QuicTransport::receiveControl( + std::vector* chunk, + std::chrono::milliseconds, + std::string* error) +{ + if (!chunk) { + if (error) *error = "control receive output must not be null"; + return TransportReceiveResult::ERROR; + } + chunk->clear(); + if (error) { + *error = "control stream receive is not supported by this transport"; + } + return TransportReceiveResult::DISCONNECTED; +} + +namespace { + +class UnavailableQuicTransport final : public QuicTransport { +public: + bool connect(const config::QuicEdgeConfig&, + std::chrono::milliseconds, + std::string* error) override + { + if (error) { + *error = "MsQuic support was not compiled (CMVR_HAS_MSQUIC is unset)"; + } + return false; + } + + void disconnect() override {} + bool isConnected() const override { return false; } + std::size_t maximumDatagramBytes() const override { return 0U; } + std::size_t maximumDatagramBatchPackets() const override { return 0U; } + + TransportSendResult sendControl(std::vector, + std::string* error) override + { + if (error) { + *error = "MsQuic support was not compiled"; + } + return TransportSendResult::DISCONNECTED; + } + + TransportReceiveResult receiveControl( + std::vector* chunk, + std::chrono::milliseconds, + std::string* error) override + { + if (!chunk) { + if (error) *error = "control receive output must not be null"; + return TransportReceiveResult::ERROR; + } + chunk->clear(); + if (error) { + *error = "MsQuic support was not compiled"; + } + return TransportReceiveResult::DISCONNECTED; + } + + TransportSendResult sendDatagramBatch(std::vector, + std::string* error) override + { + if (error) { + *error = "MsQuic support was not compiled"; + } + return TransportSendResult::DISCONNECTED; + } +}; + +} // namespace + +bool hasCompiledMsQuicSupport() +{ +#ifdef CMVR_HAS_MSQUIC + return true; +#else + return false; +#endif +} + +std::unique_ptr createDefaultQuicTransport( + const std::size_t send_queue_depth) +{ +#ifdef CMVR_HAS_MSQUIC + return createMsQuicTransport(send_queue_depth); +#else + (void)send_queue_depth; + return std::make_unique(); +#endif +} + +} // namespace cmvr::quic_edge diff --git a/cmvr-es/service/quic_edge/tests/quic_edge_protocol_test.cpp b/cmvr-es/service/quic_edge/tests/quic_edge_protocol_test.cpp new file mode 100644 index 00000000..bf9b4b35 --- /dev/null +++ b/cmvr-es/service/quic_edge/tests/quic_edge_protocol_test.cpp @@ -0,0 +1,694 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmvr/quic_edge/v1/quic_edge.pb.h" +#include "manager/media_source_hub/include/media_source_hub.h" +#include "service/quic_edge/include/control_framing.h" +#include "service/quic_edge/include/datagram_packetizer.h" +#include "service/quic_edge/include/quic_edge_service.h" + +namespace { + +#define CHECK_TRUE(expression) \ + do { \ + if (!(expression)) { \ + std::cerr << "CHECK failed at line " << __LINE__ << ": " \ + << #expression << '\n'; \ + return false; \ + } \ + } while (false) + +using namespace cmvr; + +class FakeTransport final : public quic_edge::QuicTransport { +public: + explicit FakeTransport(const bool accept_registration = true, + const bool acknowledge_heartbeats = true, + const bool valid_heartbeat_session = true, + const std::size_t media_session_would_block_count = 0U) + : accept_registration_(accept_registration), + acknowledge_heartbeats_(acknowledge_heartbeats), + valid_heartbeat_session_(valid_heartbeat_session), + media_session_would_block_count_(media_session_would_block_count) + { + } + + bool connect(const config::QuicEdgeConfig&, + std::chrono::milliseconds, + std::string*) override + { + connected_.store(true); + ++connect_count_; + { + std::lock_guard lock(mutex_); + connect_times_.push_back(std::chrono::steady_clock::now()); + } + condition_.notify_all(); + return true; + } + + void disconnect() override + { + connected_.store(false); + condition_.notify_all(); + } + bool isConnected() const override { return connected_.load(); } + std::size_t maximumDatagramBytes() const override { return 1200U; } + std::size_t maximumDatagramBatchPackets() const override { return 32U; } + + quic_edge::TransportSendResult sendControl( + std::vector message, std::string*) override + { + if (!connected_.load()) return quic_edge::TransportSendResult::DISCONNECTED; + std::vector> frames; + std::string decode_error; + quic_edge::ControlFrameDecoder decoder(1024U * 1024U); + if (!decoder.push(message, &frames, &decode_error) || frames.size() != 1U) { + return quic_edge::TransportSendResult::ERROR; + } + cmvr::quic_edge::v1::EdgeControlEnvelope envelope; + if (!envelope.ParseFromArray(frames.front().data(), + static_cast(frames.front().size()))) { + return quic_edge::TransportSendResult::ERROR; + } + { + std::lock_guard lock(mutex_); + if (envelope.has_media_session_open() && + media_session_would_block_count_ != 0U) { + --media_session_would_block_count_; + return quic_edge::TransportSendResult::WOULD_BLOCK; + } + edge_message_sequences_.push_back(envelope.message_sequence()); + controls_.push_back(std::move(message)); + if (envelope.has_node_register_request()) { + const auto& request = envelope.node_register_request(); + last_registered_node_id_ = request.node().node_id(); + last_grpc_endpoint_port_ = request.node().grpc_endpoint().port(); + last_interface_count_ = request.node().local_interfaces_size(); + cmvr::quic_edge::v1::EdgeControlEnvelope response; + response.set_protocol_version(quic_edge::kProtocolVersion); + response.set_message_sequence(server_message_sequence_++); + auto* registration = response.mutable_node_register_response(); + registration->set_accepted(accept_registration_); + registration->set_session_id( + accept_registration_ ? "test-session" : ""); + registration->set_message( + accept_registration_ ? "accepted" : "rejected for test"); + registration->set_heartbeat_interval_ms(250U); + registration->set_observed_source_ip("203.0.113.10"); + enqueueEnvelopeLocked(response); + } else if (envelope.has_node_heartbeat() && acknowledge_heartbeats_) { + const auto& heartbeat = envelope.node_heartbeat(); + cmvr::quic_edge::v1::EdgeControlEnvelope response; + response.set_protocol_version(quic_edge::kProtocolVersion); + response.set_message_sequence(server_message_sequence_++); + auto* ack = response.mutable_node_heartbeat_ack(); + ack->set_accepted(true); + ack->set_acknowledged_sequence(heartbeat.sequence()); + ack->set_session_id(valid_heartbeat_session_ + ? heartbeat.session_id() : ""); + ack->set_observed_source_ip("203.0.113.11"); + enqueueEnvelopeLocked(response); + } + } + condition_.notify_all(); + return quic_edge::TransportSendResult::QUEUED; + } + + quic_edge::TransportReceiveResult receiveControl( + std::vector* chunk, + const std::chrono::milliseconds timeout, + std::string*) override + { + if (!chunk) return quic_edge::TransportReceiveResult::ERROR; + std::unique_lock lock(mutex_); + if (control_receive_queue_.empty() && connected_.load()) { + condition_.wait_for(lock, timeout, [this]() { + return !control_receive_queue_.empty() || !connected_.load(); + }); + } + if (!control_receive_queue_.empty()) { + *chunk = std::move(control_receive_queue_.front()); + control_receive_queue_.pop_front(); + return quic_edge::TransportReceiveResult::DATA; + } + return connected_.load() + ? quic_edge::TransportReceiveResult::TIMEOUT + : quic_edge::TransportReceiveResult::DISCONNECTED; + } + + quic_edge::TransportSendResult sendDatagramBatch( + std::vector packets, std::string*) override + { + if (!connected_.load()) return quic_edge::TransportSendResult::DISCONNECTED; + std::lock_guard lock(mutex_); + datagrams_.push_back(std::move(packets)); + return quic_edge::TransportSendResult::QUEUED; + } + + std::size_t datagramBatchCount() const + { + std::lock_guard lock(mutex_); + return datagrams_.size(); + } + + std::size_t controlCount() const + { + std::lock_guard lock(mutex_); + return controls_.size(); + } + + bool controlSequencesStrictlyIncreasing() const + { + std::lock_guard lock(mutex_); + for (std::size_t index = 1U; + index < edge_message_sequences_.size(); ++index) { + if (edge_message_sequences_[index] <= + edge_message_sequences_[index - 1U]) { + return false; + } + } + return !edge_message_sequences_.empty(); + } + + std::vector datagramFrameSequences() const + { + std::lock_guard lock(mutex_); + std::vector sequences; + for (const auto& batch : datagrams_) { + if (!batch.empty()) sequences.push_back(batch.front().header.frame_sequence); + } + return sequences; + } + + std::uint64_t connectCount() const { return connect_count_.load(); } + + std::vector connectIntervals() const + { + std::lock_guard lock(mutex_); + std::vector intervals; + for (std::size_t index = 1U; index < connect_times_.size(); ++index) { + intervals.push_back(std::chrono::duration_cast( + connect_times_[index] - connect_times_[index - 1U])); + } + return intervals; + } + + std::string lastRegisteredNodeId() const + { + std::lock_guard lock(mutex_); + return last_registered_node_id_; + } + + std::uint32_t lastGrpcEndpointPort() const + { + std::lock_guard lock(mutex_); + return last_grpc_endpoint_port_; + } + + int lastInterfaceCount() const + { + std::lock_guard lock(mutex_); + return last_interface_count_; + } + +private: + void enqueueEnvelopeLocked( + const cmvr::quic_edge::v1::EdgeControlEnvelope& envelope) + { + std::string serialized; + if (!envelope.SerializeToString(&serialized)) return; + std::vector framed; + std::string error; + if (quic_edge::ControlFrameEncoder::encode( + reinterpret_cast(serialized.data()), + serialized.size(), 1024U * 1024U, &framed, &error)) { + const auto split = framed.size() / 2U; + control_receive_queue_.emplace_back( + framed.begin(), framed.begin() + static_cast(split)); + control_receive_queue_.emplace_back( + framed.begin() + static_cast(split), framed.end()); + } + } + + mutable std::mutex mutex_; + std::condition_variable condition_; + std::atomic connected_{false}; + std::atomic connect_count_{0}; + bool accept_registration_{true}; + bool acknowledge_heartbeats_{true}; + bool valid_heartbeat_session_{true}; + std::size_t media_session_would_block_count_{0U}; + std::uint64_t server_message_sequence_{0}; + std::string last_registered_node_id_; + std::uint32_t last_grpc_endpoint_port_{0}; + int last_interface_count_{0}; + std::vector> controls_; + std::deque> control_receive_queue_; + std::vector> datagrams_; + std::vector edge_message_sequences_; + std::vector connect_times_; +}; + +config::QuicEdgeConfig validConfig(const std::string& source_track_id) +{ + config::QuicEdgeConfig config; + config.set_id("quic-test"); + config.set_enable(true); + config.set_server_host("127.0.0.1"); + config.set_server_port(4433); + config.set_alpn("cmvr-quic-edge/1"); + config.set_node_id("test-node"); + config.set_software_version("test-version"); + config.set_grpc_endpoint_host("auto"); + config.set_grpc_endpoint_port(50052U); + config.set_grpc_endpoint_tls(false); + config.set_heartbeat_interval_ms(250U); + config.set_control_response_timeout_ms(100U); + config.set_include_loopback_interfaces(true); + config.mutable_tls()->set_allow_insecure(true); + config.mutable_reconnect()->set_initial_delay_ms(5); + config.mutable_reconnect()->set_maximum_delay_ms(20); + config.mutable_reconnect()->set_multiplier(2.0); + config.mutable_reconnect()->set_connect_timeout_ms(50); + config.set_maximum_datagram_bytes(1200); + config.set_maximum_control_frame_bytes(4096); + config.set_maximum_frame_bytes(32 * 1024); + config.set_datagram_send_queue_depth(32); + config.set_media_poll_interval_ms(1); + auto* track = config.add_tracks(); + track->set_track_id(7); + track->set_source_kind(config::QuicEdgeTrackConfig::SOURCE_KIND_CAMERA); + track->set_device_id("camera-test"); + track->set_source_track_id(source_track_id); + track->set_enable(true); + return config; +} + +config::QuicEdgeConfig validPresenceOnlyConfig() +{ + auto config = validConfig("unused/video/color"); + config.clear_tracks(); + return config; +} + +media::TrackDescriptorPtr videoDescriptor(const std::string& id) +{ + media::TrackDescriptor::Config config; + config.id = id; + config.source_id = "camera-test"; + config.kind = media::MediaKind::VIDEO; + config.codec = media::Codec::H264; + config.payload_format = media::PayloadFormat::ANNEX_B; + config.time_base = {1, 90000}; + config.width = 640; + config.height = 480; + config.nominal_rate = 30; + config.generation = 0x0000000200000003ULL; + return media::makeTrackDescriptor(std::move(config)); +} + +bool waitUntil(const std::function& predicate) +{ + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(2); + while (std::chrono::steady_clock::now() < deadline) { + if (predicate()) return true; + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } + return predicate(); +} + +bool testControlFraming() +{ + const std::vector payload{1, 2, 3, 4, 5}; + std::vector encoded; + std::string error; + CHECK_TRUE(quic_edge::ControlFrameEncoder::encode( + payload, 64, &encoded, &error)); + quic_edge::ControlFrameDecoder decoder(64); + std::vector> decoded; + CHECK_TRUE(decoder.push(encoded.data(), 2, &decoded, &error)); + CHECK_TRUE(decoded.empty()); + CHECK_TRUE(decoder.push(encoded.data() + 2, encoded.size() - 2, + &decoded, &error)); + CHECK_TRUE(decoded.size() == 1 && decoded.front() == payload); + return true; +} + +bool testPacketizer() +{ + auto descriptor = videoDescriptor("camera-test/video/color"); + media::MediaFrame::Config frame_config; + frame_config.descriptor = descriptor; + frame_config.payload.resize(2500, 0x5a); + frame_config.sequence = 11; + frame_config.capture_time_ns = 1234567000ULL; + frame_config.key_frame = true; + auto frame = media::makeMediaFrame(std::move(frame_config)); + + quic_edge::DatagramPacketizer packetizer(100); + std::vector packets; + std::string error; + const std::uint32_t token = + quic_edge::descriptorGenerationToken(descriptor->generation); + CHECK_TRUE(packetizer.packetize(*frame, 7, token, 99, 11, true, 1200, + &packets, &error)); + CHECK_TRUE(packets.size() == 3); + std::size_t payload_bytes = 0; + for (std::size_t index = 0; index < packets.size(); ++index) { + quic_edge::DatagramHeader header; + CHECK_TRUE(quic_edge::DatagramPacketizer::decodeHeader( + packets[index].bytes, &header, &error)); + CHECK_TRUE(header.track_id == 7 && header.session_epoch == 99); + CHECK_TRUE(header.fragment_index == index && header.fragment_count == 3); + CHECK_TRUE((header.flags & quic_edge::DATAGRAM_FLAG_KEY_FRAME) != 0); + CHECK_TRUE((header.flags & quic_edge::DATAGRAM_FLAG_DISCONTINUITY) != 0); + payload_bytes += header.payload_size; + } + CHECK_TRUE(payload_bytes == frame->size()); + return true; +} + +bool testServiceWithSharedHub() +{ + const std::string track_id = "camera-test/video/color"; + media::MediaSourceHub hub; + media::MediaSourceHub::FrameSink sink; + std::mutex sink_mutex; + std::atomic source_started{false}; + std::atomic keyframe_requests{0}; + media::MediaSourceHub::SourceCallbacks callbacks; + callbacks.start = [&](const media::MediaSourceHub::FrameSink& value, + const media::MediaSourceHub::CancelPredicate&) { + std::lock_guard lock(sink_mutex); + sink = value; + source_started.store(true); + return true; + }; + callbacks.stop = [&]() { source_started.store(false); }; + callbacks.request_key_frame = [&]() { + ++keyframe_requests; + return true; + }; + auto descriptor = videoDescriptor(track_id); + CHECK_TRUE(hub.registerSource(descriptor, std::move(callbacks), 8)); + + auto transport = std::make_unique(true, true, true, 1U); + FakeTransport* transport_view = transport.get(); + quic_edge::QuicEdgeService service( + validConfig(track_id), std::move(transport), hub); + std::string error; + CHECK_TRUE(service.initialize(&error)); + CHECK_TRUE(service.start(&error)); + CHECK_TRUE(waitUntil([&]() { return source_started.load(); })); + CHECK_TRUE(waitUntil([&]() { + return service.stats().registrations_accepted == 1U && + service.stats().heartbeats_acknowledged >= 1U; + })); + CHECK_TRUE(transport_view->lastRegisteredNodeId() == "test-node"); + CHECK_TRUE(transport_view->lastGrpcEndpointPort() == 50052U); + CHECK_TRUE(transport_view->connectCount() == 1U); + + media::MediaFrame::Config frame_config; + frame_config.descriptor = descriptor; + frame_config.payload.resize(1800, 0x11); + frame_config.sequence = 1; + frame_config.capture_time_ns = 1000000; + frame_config.key_frame = true; + media::MediaSourceHub::FrameSink publisher; + { + std::lock_guard lock(sink_mutex); + publisher = sink; + } + CHECK_TRUE(static_cast(publisher)); + publisher(media::makeMediaFrame(std::move(frame_config))); + CHECK_TRUE(waitUntil([&]() { return transport_view->datagramBatchCount() == 1; })); + CHECK_TRUE(service.stats().frames_queued == 1); + CHECK_TRUE(keyframe_requests.load() != 0); + + // Codec initialization bytes may be learned at the first encoder IDR + // without changing the source generation. The reliable descriptor must be + // refreshed rather than treating this legal enrichment as a source error. + auto enriched_descriptor_config = media::TrackDescriptor::Config{}; + enriched_descriptor_config.id = track_id; + enriched_descriptor_config.source_id = "camera-test"; + enriched_descriptor_config.kind = media::MediaKind::VIDEO; + enriched_descriptor_config.codec = media::Codec::H264; + enriched_descriptor_config.payload_format = media::PayloadFormat::ANNEX_B; + enriched_descriptor_config.time_base = {1, 90000}; + enriched_descriptor_config.width = 640; + enriched_descriptor_config.height = 480; + enriched_descriptor_config.nominal_rate = 30; + enriched_descriptor_config.generation = descriptor->generation; + enriched_descriptor_config.codec_config = {0, 0, 0, 1, 0x67}; + auto enriched_descriptor = + media::makeTrackDescriptor(std::move(enriched_descriptor_config)); + media::MediaFrame::Config enriched_frame; + enriched_frame.descriptor = std::move(enriched_descriptor); + enriched_frame.payload.resize(256, 0x22); + // Simulate a source restart that resets its device-local sequence. The + // QUIC wire sequence must remain monotonic inside the media epoch. + enriched_frame.sequence = 1; + enriched_frame.capture_time_ns = 2000000; + enriched_frame.key_frame = true; + enriched_frame.discontinuity = true; + publisher(media::makeMediaFrame(std::move(enriched_frame))); + CHECK_TRUE(waitUntil([&]() { return transport_view->datagramBatchCount() == 2; })); + const auto wire_sequences = transport_view->datagramFrameSequences(); + CHECK_TRUE(wire_sequences.size() == 2U && + wire_sequences[0] == 1U && wire_sequences[1] == 2U); + CHECK_TRUE(transport_view->controlCount() >= 3); + CHECK_TRUE(transport_view->controlSequencesStrictlyIncreasing()); + CHECK_TRUE(service.state() != quic_edge::QuicEdgeServiceState::FAILED); + service.stop(); + return true; +} + +bool testMissingInjectedSourceRetriesSafely() +{ + media::MediaSourceHub hub; + auto transport = std::make_unique(); + FakeTransport* transport_view = transport.get(); + quic_edge::QuicEdgeService service( + validConfig("missing/video/color"), std::move(transport), hub); + std::string error; + CHECK_TRUE(service.initialize(&error)); + CHECK_TRUE(service.start(&error)); + CHECK_TRUE(waitUntil([&]() { return service.stats().source_errors >= 2; })); + CHECK_TRUE(service.state() == quic_edge::QuicEdgeServiceState::ONLINE); + CHECK_TRUE(service.status().registered); + CHECK_TRUE(service.status().active_media_tracks == 0U); + CHECK_TRUE(transport_view->connectCount() == 1U); + service.stop(); + return true; +} + +bool testPresenceOnlyWithoutMedia() +{ + media::MediaSourceHub hub; + auto transport = std::make_unique(); + FakeTransport* transport_view = transport.get(); + quic_edge::QuicEdgeService service( + validPresenceOnlyConfig(), std::move(transport), hub); + std::string error; + CHECK_TRUE(service.initialize(&error)); + CHECK_TRUE(service.start(&error)); + CHECK_TRUE(waitUntil([&]() { + return service.stats().registrations_accepted == 1U && + service.stats().heartbeats_acknowledged >= 1U; + })); + const auto status = service.status(); + CHECK_TRUE(status.registered); + CHECK_TRUE(status.session_id == "test-session"); + CHECK_TRUE(status.observed_source_ip == "203.0.113.11"); + CHECK_TRUE(status.active_media_tracks == 0U); + CHECK_TRUE(transport_view->datagramBatchCount() == 0U); + CHECK_TRUE(transport_view->connectCount() == 1U); + CHECK_TRUE(service.state() == quic_edge::QuicEdgeServiceState::ONLINE); + service.stop(); + return true; +} + +bool testHeartbeatTimeoutReconnectsWithoutTaskFailure() +{ + media::MediaSourceHub hub; + auto transport = std::make_unique(true, false); + FakeTransport* transport_view = transport.get(); + auto config = validPresenceOnlyConfig(); + config.set_control_response_timeout_ms(20U); + quic_edge::QuicEdgeService service(config, std::move(transport), hub); + std::string error; + CHECK_TRUE(service.initialize(&error)); + CHECK_TRUE(service.start(&error)); + CHECK_TRUE(waitUntil([&]() { + return service.stats().heartbeat_timeouts >= 1U && + transport_view->connectCount() >= 2U; + })); + CHECK_TRUE(service.state() != quic_edge::QuicEdgeServiceState::FAILED); + service.stop(); + return true; +} + +bool testRegistrationRejectionBacksOff() +{ + media::MediaSourceHub hub; + auto transport = std::make_unique(false, true); + FakeTransport* transport_view = transport.get(); + auto config = validPresenceOnlyConfig(); + config.mutable_reconnect()->set_initial_delay_ms(20U); + config.mutable_reconnect()->set_maximum_delay_ms(80U); + quic_edge::QuicEdgeService service( + config, std::move(transport), hub); + std::string error; + CHECK_TRUE(service.initialize(&error)); + CHECK_TRUE(service.start(&error)); + CHECK_TRUE(waitUntil([&]() { + return service.stats().registrations_rejected >= 3U && + transport_view->connectCount() >= 4U; + })); + CHECK_TRUE(service.stats().heartbeats_sent == 0U); + const auto intervals = transport_view->connectIntervals(); + CHECK_TRUE(intervals.size() >= 3U); + CHECK_TRUE(intervals[0].count() >= 15); + CHECK_TRUE(intervals[1].count() >= 35); + CHECK_TRUE(intervals[2].count() >= 70); + CHECK_TRUE(service.state() != quic_edge::QuicEdgeServiceState::FAILED); + service.stop(); + return true; +} + +bool testHeartbeatAckRequiresSessionId() +{ + media::MediaSourceHub hub; + auto transport = std::make_unique(true, true, false); + FakeTransport* transport_view = transport.get(); + quic_edge::QuicEdgeService service( + validPresenceOnlyConfig(), std::move(transport), hub); + std::string error; + CHECK_TRUE(service.initialize(&error)); + CHECK_TRUE(service.start(&error)); + CHECK_TRUE(waitUntil([&]() { + return transport_view->connectCount() >= 2U && + service.stats().heartbeats_sent >= 2U; + })); + CHECK_TRUE(service.stats().heartbeats_acknowledged == 0U); + CHECK_TRUE(service.state() != quic_edge::QuicEdgeServiceState::FAILED); + service.stop(); + return true; +} + +bool testSlowMediaStartDoesNotBlockHeartbeat() +{ + const std::string track_id = "slow-camera/video/color"; + media::MediaSourceHub hub; + std::atomic start_entered{false}; + std::atomic start_exited{false}; + std::atomic release_start{false}; + media::MediaSourceHub::SourceCallbacks callbacks; + callbacks.start = [&](const media::MediaSourceHub::FrameSink&, + const media::MediaSourceHub::CancelPredicate& cancelled) { + start_entered.store(true); + while (!release_start.load() && !cancelled()) { + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } + start_exited.store(true); + return !cancelled(); + }; + callbacks.stop = []() {}; + CHECK_TRUE(hub.registerSource( + videoDescriptor(track_id), std::move(callbacks), 8U)); + + auto transport = std::make_unique(); + FakeTransport* transport_view = transport.get(); + quic_edge::QuicEdgeService service( + validConfig(track_id), std::move(transport), hub); + std::string error; + CHECK_TRUE(service.initialize(&error)); + CHECK_TRUE(service.start(&error)); + + const bool entered = waitUntil([&]() { return start_entered.load(); }); + const bool heartbeat_while_blocked = entered && waitUntil([&]() { + return service.stats().heartbeats_acknowledged >= 1U; + }); + const bool registered_while_blocked = service.status().registered; + auto stop_future = std::async(std::launch::async, [&service] { + service.stop(); + }); + const bool stop_while_start_blocked = + stop_future.wait_for(std::chrono::milliseconds(500)) == + std::future_status::ready; + release_start.store(true); + stop_future.get(); + const bool start_cancelled = waitUntil([&]() { return start_exited.load(); }); + + CHECK_TRUE(entered); + CHECK_TRUE(heartbeat_while_blocked); + CHECK_TRUE(registered_while_blocked); + CHECK_TRUE(stop_while_start_blocked); + CHECK_TRUE(start_cancelled); + CHECK_TRUE(transport_view->connectCount() == 1U); + CHECK_TRUE(transport_view->controlSequencesStrictlyIncreasing()); + return true; +} + +bool testLifecycleStateGuards() +{ + media::MediaSourceHub hub; + { + auto transport = std::make_unique(); + quic_edge::QuicEdgeService service( + validPresenceOnlyConfig(), std::move(transport), hub); + service.stop(); + CHECK_TRUE(service.state() == + quic_edge::QuicEdgeServiceState::UNINITIALIZED); + std::string error; + CHECK_TRUE(!service.start(&error)); + } + + { + auto transport = std::make_unique(); + quic_edge::QuicEdgeService service( + validPresenceOnlyConfig(), std::move(transport), hub); + std::string error; + CHECK_TRUE(service.initialize(&error)); + CHECK_TRUE(service.start(&error)); + CHECK_TRUE(waitUntil([&]() { + return service.stats().registrations_accepted == 1U; + })); + error.clear(); + CHECK_TRUE(!service.initialize(&error)); + CHECK_TRUE(!error.empty()); + service.stop(); + } + return true; +} + +} // namespace + +int main() +{ + if (!testControlFraming() || !testPacketizer() || + !testServiceWithSharedHub() || !testMissingInjectedSourceRetriesSafely() || + !testPresenceOnlyWithoutMedia() || + !testHeartbeatTimeoutReconnectsWithoutTaskFailure() || + !testRegistrationRejectionBacksOff() || + !testHeartbeatAckRequiresSessionId() || + !testSlowMediaStartDoesNotBlockHeartbeat() || + !testLifecycleStateGuards()) { + return 1; + } + std::cout << "quic_edge_protocol_test: PASS\n"; + return 0; +} diff --git a/cmvr-es/simulate/README.md b/cmvr-es/simulate/README.md new file mode 100644 index 00000000..d3933007 --- /dev/null +++ b/cmvr-es/simulate/README.md @@ -0,0 +1,58 @@ +# Simulate 模块开发指南 + +`simulate/` 保存仿真引擎适配和可视化辅助能力。当前实现主要是 MuJoCo viewer；模拟摄像头、电机和机械臂仍通过 `devices/` 中相同的设备抽象接入系统。 + +返回[项目总览](../../README.md)。 + +## 职责边界 + +`simulate/` 负责： + +- 仿真引擎上下文、模型和 viewer； +- 引擎事件循环、渲染和 UI adapter； +- 仿真资源加载的通用辅助能力。 + +`devices/` 负责： + +- 将仿真传感器或执行器实现为 `AbstractDevice` 子类； +- 对外提供与真机一致的设备语义； +- 参与 DeviceFactory、DeviceManager 和 gRPC/媒体接入。 + +不要让 service 或 task 直接调用 MuJoCo API；它们应通过设备抽象工作。 + +## 当前 CMake + +- [`CMakeLists.txt`](CMakeLists.txt) 加载 `mujoco/`； +- [`mujoco/CMakeLists.txt`](mujoco/CMakeLists.txt) 加载 viewer； +- target `cmvr_es::mujoco_viewer` 链接 MuJoCo、GLFW 和 OpenGL。 + +## 新增仿真引擎 + +1. 新建 `simulate//`； +2. 将 engine context 和资源所有权封装在独立类中； +3. 提供明确 `initialize/start/step/stop` 生命周期； +4. 通过 adapter 向模拟设备暴露最小能力； +5. 在 `devices/` 中实现对应模拟后端； +6. 在设备 config proto 的 `oneof backend` 增加配置； +7. 更新两侧 CMake； +8. 提供 headless 测试，并将图形测试单独标记。 + +## 线程和资源 + +- 明确由哪个线程调用 engine step 和 render； +- 不在多个线程无锁访问引擎状态； +- viewer 关闭不能悬挂设备回调； +- 设备停止后不得继续读取已释放的仿真模型； +- 模型路径通过 `ConfigHelper::resolveResourceFile()` 解析； +- headless 环境不能强制创建 OpenGL 窗口； +- 固定或记录仿真 timestep，避免控制周期与仿真周期混淆。 + +## 测试建议 + +- 模型加载失败和资源路径错误； +- headless 初始化、step 和停止； +- 重复启动/停止； +- 模拟设备与真机抽象的行为一致性； +- 时间步、单位、关节顺序和坐标系； +- viewer 关闭与后台设备线程并发； +- 测试结束后无线程、窗口和引擎资源泄漏。 diff --git a/cmvr-es/task/README.md b/cmvr-es/task/README.md new file mode 100644 index 00000000..aa6fc735 --- /dev/null +++ b/cmvr-es/task/README.md @@ -0,0 +1,184 @@ +# Task 模块开发指南 + +`task/` 保存由 TaskManager 管理的可运行模块。任务分为周期调度任务和自持线程/事件循环的服务任务。 + +返回[项目总览](../../README.md)。 + +## Task 契约 + +所有任务实现 [`task.h`](task.h) 中的接口： + +- `id()`：配置和管理器使用的稳定唯一 ID； +- `runMode()`：周期任务或服务任务； +- `init()`：配置校验和本地资源准备； +- `start()`：启动执行，必须快速返回； +- `step(dt)`：周期任务的一次计算； +- `stop()`：幂等停止并回收任务拥有的资源； +- `state()` 和状态辅助接口：线程安全地暴露状态； +- `detailStatusString()`：提供可诊断状态，不包含密钥。 + +析构函数应安全调用 `stop()`。异步状态、错误和统计必须使用 mutex/atomic 保护。 + +## 运行模式 + +### PERIODIC_STEP + +- 由 TaskManager scheduler 调用 `step()`； +- 配置 `control_period_s` 必须是有限正数； +- 只有 task state 为 `RUNNING` 时执行 step； +- 所有周期任务共享一个 scheduler 线程； +- step 不能做阻塞网络 I/O 或无界计算； +- step 返回失败时应更新任务状态和可诊断错误； +- 不要在 step 内同步停止 TaskManager。 + +### BLOCKING_SERVICE + +- TaskManager 不调用其 `step()`； +- start 必须创建自己的 server/worker 后快速返回； +- stop 必须关闭 server、唤醒等待并 join 线程； +- 不能把 `BLOCKING_SERVICE` 理解为 start 可以永久阻塞。 + +`GrpcServerTask` 和 `QuicEdgeTask` 是当前服务任务参考实现。 + +## 新增任务的完整链路 + +### 1. 配置 Proto + +在 `protos/cmvr/config/` 新增： + +```protobuf +message ExampleTaskConfig { + string id = 1; +} + +message ExampleTaskRootConfig { + ExampleTaskConfig example_task = 1; +} +``` + +在 `TaskConfigEntry::TaskType` 中使用新的、稳定的枚举值。当前 `2` 和 `4` 已 reserved，不得复用。 + +### 2. 实现目录 + +```text +task/example_task/ +├── CMakeLists.txt +├── include/example_task.h +├── src/example_task.cpp +└── tests/example_task_test.cpp +``` + +实现 `Task` 全部接口。持续运行任务的典型状态转换为： + +```text +UNINITIALIZED -> IDLE -> RUNNING -> STOPPED + └-------> FAILED +``` + +持续周期任务必须在 start 后进入 `RUNNING`，否则 scheduler 不会调用 step。命令驱动任务可以在 start 后保持 `IDLE`，收到外部命令后再进入 `RUNNING`，当前 TouchScreenTask 就采用这种方式。 + +init、start 或 step 都可能进入 `FAILED`，一次性任务也可以进入 `SUCCEEDED`。失败后是否允许重新 init/start 必须在类注释和测试中明确。 + +### 3. Creator 与注册 + +creator 应： + +1. 校验 manager entry ID 和 config file； +2. 使用 `ConfigHelper` 加载 root config； +3. 校验子配置 ID 与 manager entry ID 一致； +4. 只创建对象，不进行耗时外部连接； +5. 返回 `nullptr` 并记录明确错误。 + +提供： + +```cpp +void registerExampleTaskFactory(); +``` + +通过 [`task_factory.h`](task_factory.h) 的 `TaskFactory::registerCreator()` 注册。 + +`registerCreator()` 对相同 TaskType 会覆盖旧 creator，当前不会报错。不要重复注册。creator 在 registry mutex 持有期间执行，因此不能从 creator 递归调用 register/create。 + +### 4. main 注册 + +在 `TaskManager::getInstance()` 之前显式调用注册函数。当前位置见 [`../main.cpp`](../main.cpp)。 + +TouchScreen 是 TaskFactory 内硬编码的历史实现；新任务优先使用显式 registry 模式。 + +### 5. TaskManager 映射 + +在 [`../manager/task_manager/src/task_manager.cpp`](../manager/task_manager/src/task_manager.cpp) 的 TaskType 字符串映射中增加新类型，否则计划日志会显示 unknown。 + +### 6. 默认配置 + +新增： + +```text +cmvr-es/config/tasks/example_task/example_task.pb.txt +``` + +并在 `config/manager/task_manager.pb.txt` 增加 entry。依赖硬件、网络或证书的新任务默认 `enable: false`。 + +配置 `run_mode` 必须和 task 的 `runMode()` 一致，否则 TaskManager 会跳过该任务。 + +### 7. CMake + +- 为任务建立独立 library target； +- 添加项目命名空间 alias； +- 在 [`CMakeLists.txt`](CMakeLists.txt) 或 [`../CMakeLists.txt`](../CMakeLists.txt) 加入子目录； +- 根 `cmvr_es` 必须链接注册函数所在 target，避免静态库未被带入； +- 测试放在 `if(BUILD_TESTING)`； +- 使用 `add_test()` 登记到 CTest。 + +参考 [`quic_edge_task/CMakeLists.txt`](quic_edge_task/CMakeLists.txt)。 + +## TaskManager 当前语义 + +- init 按配置顺序执行； +- init 失败只跳过该任务，进程继续； +- ID 重复会跳过后加入项； +- run mode 不匹配会跳过； +- `TASK_RUN_MODE_UNKNOWN` 当前会记录错误并退化为 `PERIODIC_STEP`，配置不得依赖该历史行为； +- start 阶段任一任务失败会停止此前已启动任务； +- 返回 false 的 task 必须自行清理此次 start 的部分资源； +- 任务保存在 unordered_map，不能依赖 start/stop 顺序； +- manager 处于 running 状态时，stop 先停止 scheduler，再调用每个任务 stop； +- 未启动或已经停止时，`stopRunTask()` 会直接返回； +- TaskManager 单例已存在时传入新配置不会热更新。 + +任务之间有依赖时，应由一个协调 task 显式管理，或增加明确依赖模型；不能依赖配置顺序碰巧变成启动顺序。 + +## 测试最低要求 + +- 缺失配置、空 ID 和 ID mismatch； +- run-mode mismatch； +- init/start/stop 状态转换； +- stop 重复调用； +- start 失败后的资源清理； +- 周期 dt、调度延迟和 step 失败； +- service worker 正常 shutdown； +- stop 发生在阻塞等待期间； +- 析构时仍处于 RUNNING； +- fake 依赖下的无设备运行。 + +单项测试： + +```bash +cmake --build build --target quic_edge_task_test +ctest \ + --test-dir build \ + -R '^quic_edge_task_test$' \ + --output-on-failure +``` + +## 提交检查 + +- [ ] 使用新的 TaskType 枚举号 +- [ ] 配置 root message、默认配置和 ID 一致 +- [ ] creator 与 main 注册均已完成 +- [ ] run mode 与配置一致 +- [ ] start 快速返回 +- [ ] stop 幂等并 join 所有线程 +- [ ] 状态查询线程安全 +- [ ] CMake target 被根程序链接 +- [ ] 测试已登记到 CTest diff --git a/cmvr-es/task/quic_edge_task/CMakeLists.txt b/cmvr-es/task/quic_edge_task/CMakeLists.txt new file mode 100644 index 00000000..009388df --- /dev/null +++ b/cmvr-es/task/quic_edge_task/CMakeLists.txt @@ -0,0 +1,27 @@ +add_library(quic_edge_task STATIC src/quic_edge_task.cpp) +target_compile_features(quic_edge_task PUBLIC cxx_std_17) +target_include_directories(quic_edge_task PUBLIC ${PROJECT_SOURCE_DIR}/cmvr-es) +target_link_libraries(quic_edge_task + PUBLIC + cmvr_es::task + cmvr_es::quic_edge_service + PRIVATE + cmvr_es::proto + cmvr_es::logging + cmvr_es::device_manager +) +add_library(cmvr_es::quic_edge_task ALIAS quic_edge_task) + +if(BUILD_TESTING) + add_executable(quic_edge_task_test tests/quic_edge_task_test.cpp) + target_compile_features(quic_edge_task_test PRIVATE cxx_std_17) + target_link_libraries(quic_edge_task_test PRIVATE cmvr_es::quic_edge_task) + add_test(NAME quic_edge_task_test COMMAND quic_edge_task_test) + if(UNIX AND NOT APPLE) + get_property(_quic_task_test_library_dirs DIRECTORY PROPERTY LINK_DIRECTORIES) + list(PREPEND _quic_task_test_library_dirs "${CMAKE_BINARY_DIR}/cmvr_compiler_runtime") + list(JOIN _quic_task_test_library_dirs ":" _quic_task_test_library_path) + set_tests_properties(quic_edge_task_test PROPERTIES + ENVIRONMENT "LD_LIBRARY_PATH=${_quic_task_test_library_path}") + endif() +endif() diff --git a/cmvr-es/task/quic_edge_task/include/quic_edge_task.h b/cmvr-es/task/quic_edge_task/include/quic_edge_task.h new file mode 100644 index 00000000..b5c760b5 --- /dev/null +++ b/cmvr-es/task/quic_edge_task/include/quic_edge_task.h @@ -0,0 +1,49 @@ +#ifndef CMVR_ES_QUIC_EDGE_TASK_H +#define CMVR_ES_QUIC_EDGE_TASK_H + +#include +#include +#include + +#include "cmvr/config/quic_edge_config/quic_edge_config.pb.h" +#include "service/quic_edge/include/quic_edge_service.h" +#include "task/task.h" + +namespace cmvr::task { + +class QuicEdgeTask final : public Task { +public: + explicit QuicEdgeTask(const config::QuicEdgeConfig& config); + ~QuicEdgeTask() override; + + const std::string& id() const override { return id_; } + TaskRunMode runMode() const override { return TaskRunMode::BLOCKING_SERVICE; } + bool init() override; + bool start() override; + bool step(double dt) override; + void stop() override; + + TaskState state() const override; + bool isBusy() const override; + bool isFinished() const override; + bool isFailed() const override; + std::string stateString() const override; + std::string detailStatusString() const override; + +private: + TaskState mappedState() const; + + config::QuicEdgeConfig config_; + std::string id_; + std::unique_ptr service_; + + mutable std::mutex mutex_; + TaskState state_{TaskState::UNINITIALIZED}; + std::string last_error_; +}; + +void registerQuicEdgeTaskFactory(); + +} // namespace cmvr::task + +#endif // CMVR_ES_QUIC_EDGE_TASK_H diff --git a/cmvr-es/task/quic_edge_task/src/quic_edge_task.cpp b/cmvr-es/task/quic_edge_task/src/quic_edge_task.cpp new file mode 100644 index 00000000..96ec5367 --- /dev/null +++ b/cmvr-es/task/quic_edge_task/src/quic_edge_task.cpp @@ -0,0 +1,200 @@ +#include "task/quic_edge_task/include/quic_edge_task.h" + +#include +#include + +#include "cmvr/config/task_manager_config/task_manager_config.pb.h" +#include "common/base/logging/logger.h" +#include "common/config/config_files.h" +#include "manager/device_manager/include/device_manager.h" +#include "task/task_factory.h" + +namespace cmvr::task { +namespace { + +std::shared_ptr createQuicEdgeTask(const config::TaskConfigEntry& entry) +{ + if (entry.id().empty() || entry.config_file().empty()) { + CMVR_LOG(ERROR) << "[QuicEdgeTask] Task id or config_file is empty"; + return nullptr; + } + config::QuicEdgeRootConfig root; + if (!ConfigHelper::loadConfigFile(entry.config_file(), root)) { + CMVR_LOG(ERROR) << "[QuicEdgeTask] Failed to load config: " + << entry.config_file(); + return nullptr; + } + config::QuicEdgeConfig config = root.quic_edge(); + if (config.id().empty() || config.id() != entry.id()) { + CMVR_LOG(ERROR) << "[QuicEdgeTask] Task ID mismatch: manager=" << entry.id() + << ", config=" << config.id(); + return nullptr; + } + if (!config.tls().ca_file().empty()) { + config.mutable_tls()->set_ca_file( + ConfigHelper::resolveConfigFile(config.tls().ca_file())); + } + if (!config.tls().certificate_file().empty()) { + config.mutable_tls()->set_certificate_file( + ConfigHelper::resolveConfigFile(config.tls().certificate_file())); + } + if (!config.tls().private_key_file().empty()) { + config.mutable_tls()->set_private_key_file( + ConfigHelper::resolveConfigFile(config.tls().private_key_file())); + } + if (config.software_version().empty()) { + config.set_software_version(device::DeviceManager::getInstance().version()); + } + return std::make_shared(config); +} + +} // namespace + +QuicEdgeTask::QuicEdgeTask(const config::QuicEdgeConfig& config) + : config_(config), id_(config.id()) +{ +} + +QuicEdgeTask::~QuicEdgeTask() +{ + stop(); +} + +bool QuicEdgeTask::init() +{ + std::lock_guard lock(mutex_); + if (id_.empty()) { + last_error_ = "QUIC edge task id is empty"; + state_ = TaskState::FAILED; + return false; + } + service_ = std::make_unique(config_); + std::string error; + if (!service_->initialize(&error)) { + last_error_ = std::move(error); + state_ = TaskState::FAILED; + return false; + } + last_error_.clear(); + state_ = TaskState::IDLE; + return true; +} + +bool QuicEdgeTask::start() +{ + std::lock_guard lock(mutex_); + if (state_ == TaskState::RUNNING) return true; + if ((state_ != TaskState::IDLE && state_ != TaskState::STOPPED) || !service_) { + last_error_ = "QUIC edge task is not initialized"; + state_ = TaskState::FAILED; + return false; + } + std::string error; + if (!service_->start(&error)) { + last_error_ = std::move(error); + state_ = TaskState::FAILED; + return false; + } + last_error_.clear(); + state_ = TaskState::RUNNING; + CMVR_LOG(INFO) << "[QuicEdgeTask] Started, id=" << id_ + << ", enabled=" << config_.enable(); + return true; +} + +bool QuicEdgeTask::step(const double dt) +{ + (void)dt; + return !isFailed(); +} + +void QuicEdgeTask::stop() +{ + std::lock_guard lock(mutex_); + // Keep ownership stable for the complete stop. init() may replace the + // unique service instance and therefore must not race a raw pointer here. + if (service_) service_->stop(); + if (state_ != TaskState::FAILED) state_ = TaskState::STOPPED; +} + +TaskState QuicEdgeTask::mappedState() const +{ + if (!service_ || state_ != TaskState::RUNNING) return state_; + return service_->state() == quic_edge::QuicEdgeServiceState::FAILED + ? TaskState::FAILED : state_; +} + +TaskState QuicEdgeTask::state() const +{ + std::lock_guard lock(mutex_); + return mappedState(); +} + +bool QuicEdgeTask::isBusy() const +{ + return state() == TaskState::RUNNING; +} + +bool QuicEdgeTask::isFinished() const +{ + return state() == TaskState::STOPPED; +} + +bool QuicEdgeTask::isFailed() const +{ + return state() == TaskState::FAILED; +} + +std::string QuicEdgeTask::stateString() const +{ + return taskStateToString(state()); +} + +std::string QuicEdgeTask::detailStatusString() const +{ + std::lock_guard lock(mutex_); + std::ostringstream output; + output << taskStateToString(mappedState()); + if (!config_.enable()) { + output << " disabled"; + return output.str(); + } + if (service_) { + const auto stats = service_->stats(); + const auto service_status = service_->status(); + output << " service=" << quic_edge::toString(service_->state()) + << " target=" << config_.server_host() << ':' << config_.server_port() + << " node_id=" << service_status.node_id + << " registered=" << service_status.registered + << " connections=" << stats.successful_connections + << " registrations=" << stats.registrations_accepted + << " heartbeat_sequence=" << service_status.heartbeat_sequence + << " heartbeat_acks=" << stats.heartbeats_acknowledged + << " media_tracks=" << service_status.active_media_tracks + << " frames=" << stats.frames_queued + << " datagrams=" << stats.datagrams_queued; + if (!service_status.session_id.empty()) { + output << " session=" << service_status.session_id; + } + if (!service_status.observed_source_ip.empty()) { + output << " observed_ip=" << service_status.observed_source_ip; + } + if (!service_status.last_media_error.empty()) { + output << " media_error=" << service_status.last_media_error; + } + const std::string service_error = service_->lastError(); + if (!service_error.empty()) output << " error=" << service_error; + } else if (!last_error_.empty()) { + output << " error=" << last_error_; + } + return output.str(); +} + +void registerQuicEdgeTaskFactory() +{ + TaskFactory::registerCreator( + config::TaskConfigEntry::TASK_TYPE_QUIC_EDGE, + createQuicEdgeTask); +} + +} // namespace cmvr::task diff --git a/cmvr-es/task/quic_edge_task/tests/quic_edge_task_test.cpp b/cmvr-es/task/quic_edge_task/tests/quic_edge_task_test.cpp new file mode 100644 index 00000000..33d896d6 --- /dev/null +++ b/cmvr-es/task/quic_edge_task/tests/quic_edge_task_test.cpp @@ -0,0 +1,28 @@ +#include + +#include "task/quic_edge_task/include/quic_edge_task.h" + +int main() +{ + cmvr::config::QuicEdgeConfig config; + config.set_id("quic-disabled-test"); + config.set_enable(false); + + cmvr::task::QuicEdgeTask task(config); + if (!task.init() || task.state() != cmvr::task::TaskState::IDLE) { + std::cerr << "disabled QUIC task failed to initialize\n"; + return 1; + } + if (!task.start() || task.state() != cmvr::task::TaskState::RUNNING || + task.detailStatusString().find("disabled") == std::string::npos) { + std::cerr << "disabled QUIC task failed to start safely\n"; + return 1; + } + task.stop(); + if (task.state() != cmvr::task::TaskState::STOPPED) { + std::cerr << "disabled QUIC task failed to stop\n"; + return 1; + } + std::cout << "quic_edge_task_test: PASS\n"; + return 0; +} diff --git a/protos/README.md b/protos/README.md new file mode 100644 index 00000000..2e75b513 --- /dev/null +++ b/protos/README.md @@ -0,0 +1,186 @@ +# Protobuf 与协议开发指南 + +`protos/` 是 CMVR-ES 配置、gRPC API、共享消息和 QUIC 控制面的契约源。生成代码由 CMake 创建，不应手工编辑或提交。 + +返回[项目总览](../README.md)。 + +## 目录职责 + +| 目录 | 职责 | +| --- | --- | +| `cmvr/api/` | gRPC service 和 command DTO | +| `cmvr/msgs/` | 设备、状态和错误等共享消息 | +| `cmvr/config/` | Proto Text 运行配置 Schema | +| `cmvr/common/` | 通用几何等跨领域消息 | +| `cmvr/quic_edge/v1/` | QUIC reliable control message 和 v1 wire 文档 | +| `rbk/` | 第三方/兼容协议 Schema | + +## C++ 生成流程 + +根 [`CMakeLists.txt`](../CMakeLists.txt)： + +1. 递归收集 `protos/**/*.proto`； +2. 使用 `protoc` 生成 `.pb.h/.pb.cc`； +3. 使用 `grpc_cpp_plugin` 生成 `.grpc.pb.h/.grpc.pb.cc`； +4. 输出到 `build/_protobuf/`； +5. 打包为 `cmvr_es::proto`。 + +`build/_protobuf/` 是构建产物，不手改、不提交。 + +当前使用 `file(GLOB_RECURSE ...)` 且没有 `CONFIGURE_DEPENDS`。新增 Proto 文件后必须重新执行 CMake configure： + +```bash +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMVR_ARCH=x86 + +cmake --build build -j"$(nproc)" +``` + +`protoc` 和 `grpc_cpp_plugin` 当前固定从 `output/bin/` 使用。干净环境首次构建先执行根 README 的工具引导步骤。 + +## 通用兼容规则 + +已发布协议中禁止： + +- 修改或复用字段号； +- 修改 enum 数值含义； +- 修改 oneof tag； +- 修改字段 wire type； +- 修改 gRPC package、service 或 method 全名； +- 将原本可选的字段改成必填语义； +- 把同一字段单位从 m 改成 mm 等隐式破坏。 + +删除字段时同时 reserved 字段号和名称： + +```protobuf +message Example { + reserved 2; + reserved "old_field"; + + string id = 1; + string new_field = 3; +} +``` + +新增字段使用新 tag。需要区分“缺失”和“零值”时使用 proto3 `optional`，并验证目标语言工具链支持。 + +破坏性 gRPC 变化应创建版本化 package/service，而不是原地改变旧方法。 + +## 配置 Proto + +配置也必须考虑已部署 `.pb.txt`： + +- 新 scalar 的零值不能自动成为危险默认值； +- C++ 必须做范围和跨字段校验； +- enum unknown、oneof 未设置应明确失败； +- root message 命名保持 `XxxRootConfig`； +- 字段注释写明单位、范围和安全语义； +- Schema 变更同步 [`../cmvr-es/config/README.md`](../cmvr-es/config/README.md) 与样例配置。 + +TextFormat 使用字段名，因此随意重命名字段同样会破坏旧配置。 + +## gRPC API + +推荐一个领域拆分为： + +```text +cmvr/api/example_command.proto +cmvr/api/example_service.proto +``` + +service 文件 import command 文件。公共 header、错误和时间戳复用 `common.proto`，不要复制出多个略有差异的定义。 + +新增 API 后还需要： + +1. 实现 C++ service； +2. 加入 service CMake target； +3. 在 GrpcServerTask 中构造和注册； +4. 生成 Java client； +5. 增加 handler、reflection 和 grpcurl 测试。 + +完整流程见 [`../cmvr-es/service/README.md`](../cmvr-es/service/README.md)。 + +`cmvr/api/test_service.proto` 当前没有实现或注册，且包含历史拼写 `TestReqeust`，不要把它作为新 API 模板。 + +## QUIC Edge v1 + +当前固定契约： + +- package：`cmvr.quic_edge.v1` +- `protocol_version = 1` +- ALPN：`cmvr-quic-edge/1` +- reliable stream：`uint32_be length + EdgeControlEnvelope` +- media DATAGRAM：固定 64 字节非 Protobuf header + payload + +完整字段和偏移见 [`cmvr/quic_edge/v1/README.md`](cmvr/quic_edge/v1/README.md)。 + +破坏 framing、固定头、状态机或时序时，必须建立新协议版本、package 和 ALPN，不能只修改 Proto。 + +新增 control envelope payload 时： + +- 使用未占用的新 oneof tag； +- 同步 edge 发送/dispatch 和状态机； +- 同步 message/session/heartbeat sequence 校验； +- 同步 Java Gateway； +- 更新 wire 文档； +- 增加 golden vector 和 fake transport 测试。 + +Edge 当前入站只实现 RegisterResponse、HeartbeatAck 和 ProtocolError。Proto 中存在的消息不等于两个方向都已实现。 + +每次 QUIC 连接建立后，Edge 的出站 control `message_sequence` 从 `0` 重新开始，因此首个 `NodeRegisterRequest.message_sequence` 当前为 `0`。Gateway 必须接受该起始值，不能假设从 `1` 开始；后续只要求同一发送方向严格递增，允许出现间隔。 + +QUIC stream 与 DATAGRAM 没有跨通道到达顺序保证。Gateway 不能假设 session/descriptor 一定先于媒体到达。 + +## Java 生成 + +仓库只自动生成 C++。Java 平台应在自己的 Gradle/Maven 工程中锁定： + +- protoc +- protobuf-java +- protoc-gen-grpc-java +- grpc-java + +并以 `protos/` 作为 import root。 + +QUIC Gateway 只需普通 Protobuf Java message： + +```bash +output/bin/protoc \ + -I protos \ + --java_out=platform-gateway/src/main/java \ + protos/cmvr/quic_edge/v1/quic_edge.proto +``` + +gRPC Java client 还需要 grpc-java plugin。当前 Proto 没有统一 `java_package` 和 `java_multiple_files`，增加这些 option 时应同步两端生成结果和平台 import。 + +浏览器不能直接消费自定义 QUIC ALPN，不应从这些 Proto 推导“浏览器可直连 Edge”。 + +## Review 与验证 + +```bash +# 重新生成并编译 C++ +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMVR_ARCH=x86 +cmake --build build -j"$(nproc)" + +# 验证当前自动测试 +ctest --test-dir build --output-on-failure + +# 验证运行时 reflection +/tmp/grpcurl -plaintext 127.0.0.1:50052 list +``` + +评审清单： + +- [ ] 新字段只使用新 tag +- [ ] 删除字段同时 reserved number 和 name +- [ ] enum 数值和语义兼容 +- [ ] 旧 pb.txt 仍可解析 +- [ ] C++ 重新 configure 并生成 +- [ ] Java/其他语言生成成功 +- [ ] service 已真正注册 +- [ ] grpcurl reflection 与预期一致 +- [ ] QUIC wire 文档和测试同步更新 +- [ ] 破坏性变化采用新版本 diff --git a/protos/cmvr/api/common.proto b/protos/cmvr/api/common.proto index 2ea6077c..5686a18a 100644 --- a/protos/cmvr/api/common.proto +++ b/protos/cmvr/api/common.proto @@ -35,6 +35,7 @@ message AudioData { MP3 = 1; AAC = 2; WAV = 3; + OPUS = 4; } bytes data = 1; int32 sample_rate = 2; diff --git a/protos/cmvr/config/quic_edge_config/quic_edge_config.proto b/protos/cmvr/config/quic_edge_config/quic_edge_config.proto new file mode 100644 index 00000000..819e50b2 --- /dev/null +++ b/protos/cmvr/config/quic_edge_config/quic_edge_config.proto @@ -0,0 +1,88 @@ +syntax = "proto3"; + +package cmvr.config; + +message QuicEdgeTlsConfig { + string ca_file = 1; + string certificate_file = 2; + string private_key_file = 3; + string server_name = 4; + + // Development-only escape hatch. Production configurations should keep + // this false and provide ca_file plus server_name. + bool allow_insecure = 5; +} + +message QuicEdgeReconnectConfig { + uint32 initial_delay_ms = 1; + uint32 maximum_delay_ms = 2; + double multiplier = 3; + uint32 jitter_percent = 4; + uint32 connect_timeout_ms = 5; +} + +message QuicEdgeTrackConfig { + enum SourceKind { + SOURCE_KIND_UNSPECIFIED = 0; + SOURCE_KIND_CAMERA = 1; + SOURCE_KIND_MICROPHONE = 2; + } + + uint32 track_id = 1; + SourceKind source_kind = 2; + string device_id = 3; + bool enable = 4; + + // Zero uses QuicEdgeConfig.maximum_frame_bytes. + uint32 max_frame_bytes = 5; + + // Protocol-neutral MediaSourceHub track ID. When empty, the service derives + // the default ID from source_kind and device_id (for example + // "right_hand_cam/video/color"). The Hub's descriptor is the authority for + // codec metadata and descriptor generation. + string source_track_id = 6; +} + +message QuicEdgeConfig { + string id = 1; + bool enable = 2; + string server_host = 3; + uint32 server_port = 4; + string alpn = 5; + string node_id = 6; + QuicEdgeTlsConfig tls = 7; + QuicEdgeReconnectConfig reconnect = 8; + + // Local safety limits. The effective DATAGRAM size is the minimum of this + // value and the peer/path value reported by the QUIC implementation. + uint32 maximum_datagram_bytes = 9; + uint32 maximum_control_frame_bytes = 10; + uint32 maximum_frame_bytes = 11; + uint32 datagram_send_queue_depth = 12; + uint32 media_poll_interval_ms = 13; + + // Node-presence fields migrated from the former gRPC edge agent. node_id may + // be "auto" when the implementation supports hostname-based resolution. + string software_version = 14; + + // Existing edge gRPC server endpoint advertised to the gateway. Empty or + // "auto" host selects a usable address from the current interface snapshot. + string grpc_endpoint_host = 15; + uint32 grpc_endpoint_port = 16; + bool grpc_endpoint_tls = 17; + + // The receiver may override heartbeat_interval_ms in NodeRegisterResponse. + // control_response_timeout_ms applies while waiting for registration and + // heartbeat acknowledgements on the reliable stream. + uint32 heartbeat_interval_ms = 18; + uint32 control_response_timeout_ms = 19; + + // An empty list, or a list with no enabled entry, is valid: node registration, + // IP reporting and heartbeat continue without a camera or microphone. + repeated QuicEdgeTrackConfig tracks = 20; + bool include_loopback_interfaces = 21; +} + +message QuicEdgeRootConfig { + QuicEdgeConfig quic_edge = 1; +} diff --git a/protos/cmvr/config/task_manager_config/task_manager_config.proto b/protos/cmvr/config/task_manager_config/task_manager_config.proto index 6c13c845..68001d40 100644 --- a/protos/cmvr/config/task_manager_config/task_manager_config.proto +++ b/protos/cmvr/config/task_manager_config/task_manager_config.proto @@ -6,8 +6,9 @@ message TaskConfigEntry { TASK_TYPE_UNKNOWN = 0; TASK_TYPE_TOUCH_SCREEN = 1; TASK_TYPE_GRPC_SERVER = 3; - reserved 2; - reserved "TASK_TYPE_ARM_CONTROL"; + TASK_TYPE_QUIC_EDGE = 5; + reserved 2, 4; + reserved "TASK_TYPE_ARM_CONTROL", "TASK_TYPE_GRPC_EDGE_AGENT"; } enum TaskRunMode { diff --git a/protos/cmvr/quic_edge/v1/README.md b/protos/cmvr/quic_edge/v1/README.md new file mode 100644 index 00000000..ceb742e9 --- /dev/null +++ b/protos/cmvr/quic_edge/v1/README.md @@ -0,0 +1,160 @@ +# QUIC edge v1 wire format + +## Scope and topology + +QUIC edge v1 carries two functions over one authenticated QUIC connection: + +- a reliable bidirectional control stream for node registration, heartbeat, + current local IP addresses, the advertised gRPC endpoint and media metadata; +- QUIC DATAGRAM payloads for lossy, real-time audio and video. + +Robot arm, AGV and other device-control APIs remain on the existing cmvr-es +gRPC server. The QUIC node descriptor advertises that server's current endpoint; +it does not move robot-control RPCs onto QUIC. + +```text +cmvr-es -- custom QUIC edge v1 --> Java receiver / edge gateway + | + +-- WebTransport/HTTP3 --> browser +``` + +A browser cannot connect to this custom QUIC ALPN directly. The gateway must +terminate v1, register the node, reassemble media fragments and expose +WebTransport (or another browser-supported media API). Browser sessions do not +participate in edge-node registration or heartbeat. + +## Build and host-safe defaults + +MsQuic is optional at configure time. It can be installed under +`dependency//third_party/msquic/` or supplied explicitly: + +```bash +cmake -S . -B build \ + -DCMVR_ENABLE_MSQUIC_BACKEND=ON \ + -DCMVR_MSQUIC_ROOT=/absolute/path/to/msquic +cmake --build build -j2 +cmake --install build +``` + +Without MsQuic the project still builds, but enabling the QUIC task fails with +an explicit error. The repository default keeps `quic_edge` disabled. The +existing inbound gRPC server remains enabled independently. + +To enable node presence without media: + +1. Configure the gateway address, ALPN `cmvr-quic-edge/1`, TLS trust, node ID, + advertised gRPC endpoint and heartbeat values in + `cmvr-es/config/tasks/quic_edge_task/quic_edge_task.pb.txt`. + Certificate paths are relative to the cmvr-es configuration root, for + example `certs/quic_gateway_ca.pem`. +2. Leave `tracks` empty, set `quic_edge.enable: true`, and enable the + `quic_edge` TaskManager entry. + +To add a media uplink, also enable the camera/microphone in DeviceManager and +add a matching `tracks` entry. A zero-track configuration is valid: unavailable +media must not suppress node registration, IP reporting or heartbeat. +`maximum_frame_bytes` must fit in one atomically admitted DATAGRAM batch; reduce +it or increase `datagram_send_queue_depth` when changing the DATAGRAM size. + +## Reliable control stream + +The edge opens one bidirectional reliable stream after the QUIC handshake. Both +directions carry a sequence of length-prefixed protobuf messages: + +```text +uint32_be protobuf_length +protobuf EdgeControlEnvelope +``` + +The length excludes the four-byte prefix. Each receiver must reject a frame +larger than its configured control-frame limit. `protocol_version` is `1`. +`message_sequence` increases independently in each direction; gaps are allowed, +but duplicates and reordering are rejected. The edge resets its outbound +control sequence on every connection, so its first `NodeRegisterRequest` +currently has `message_sequence=0`; gateways must not assume the sequence starts +at 1. Heartbeat uses a separate sequence so its acknowledgement remains +explicit. + +The legal session order is: + +1. The edge sends `NodeRegisterRequest` as its first application message after + every QUIC connect or reconnect. It includes `node_id`, `boot_id`, software + version, the current IPv4/IPv6 interface snapshot and advertised gRPC endpoint. +2. The gateway replies with `NodeRegisterResponse`. Media and heartbeat must not + start until `accepted=true` and a non-empty `session_id` are received. +3. The edge sends `NodeHeartbeat` at the negotiated interval. The gateway returns + `NodeHeartbeatAck` with the same `session_id` and exact + `acknowledged_sequence`. A missing, rejected or mismatched ACK causes the edge + to reconnect and register again after its configured backoff. +4. If media tracks are available, the edge sends `MediaSessionOpen`, binding its + `session_epoch` to the accepted `session_id`, followed by track descriptors. + The edge calls reliable descriptor send before DATAGRAM send for a new + generation. QUIC does not guarantee arrival order across a reliable stream + and DATAGRAMs, so a gateway must boundedly buffer or drop media whose session + or generation metadata has not arrived yet. +5. The gateway may send `ProtocolError`; a fatal error closes the connection. + The current edge receive path otherwise accepts only `NodeRegisterResponse` + and `NodeHeartbeatAck`. Although `MediaSessionClose` is present in the v1 + schema, the current edge neither sends it nor accepts it inbound. Gateways + must not depend on that message until both sides implement and test it. + +The edge sends a fresh local-interface snapshot in every heartbeat, so an IP +change is reported without a second protocol. These addresses are edge claims. +`observed_source_ip` is authoritative for the public/NAT-facing address and must +be derived by the gateway from the authenticated QUIC peer, never copied from a +client field. `heartbeat_interval_ms=0` in the registration response means the +edge retains its configured interval. + +DATAGRAM negotiation is required only when at least one media track is enabled. +The reliable registration and heartbeat path remains valid for a zero-track +node or while all media sources are unavailable. + +Media source registration, device start/keyframe callbacks and DATAGRAM +packetization run on a dedicated media worker. Reliable registration and +heartbeat therefore continue while a media source is slow or unavailable; +the reliable send path serializes heartbeat and media metadata so envelope +sequence order remains strict. + +`MediaSourceHub` is protocol-neutral and gives each adapter an independent, +single-consumer subscription cursor. Its source-start callback receives a +cancellation predicate and must check it around potentially blocking device +startup. QUIC reconnect/stop and gRPC client cancellation can therefore abandon +startup without blocking presence or process shutdown. Ring overflow keeps the +newest bounded set of immutable frames, advances a slow cursor to the oldest +retained frame, and reports the exact dropped count so protocol adapters can +mark a discontinuity and request a new video keyframe. + +## Media DATAGRAM wire format + +Every media DATAGRAM starts with this fixed 64-byte, network-byte-order header: + +| Offset | Size | Field | +|---:|---:|---| +| 0 | 4 | magic `CMQD` (`0x434d5144`) | +| 4 | 1 | protocol version (`1`) | +| 5 | 1 | media kind (`1` video, `2` audio) | +| 6 | 2 | flags | +| 8 | 2 | header size (`64`) | +| 10 | 2 | fragment index | +| 12 | 2 | fragment count | +| 14 | 2 | payload size | +| 16 | 4 | track ID | +| 20 | 4 | codec generation token | +| 24 | 8 | media session epoch | +| 32 | 8 | DATAGRAM packet sequence | +| 40 | 8 | media frame sequence | +| 48 | 8 | monotonic capture timestamp in microseconds | +| 56 | 4 | complete frame size | +| 60 | 4 | fragment byte offset in the complete frame | + +Flag bit 0 marks a video keyframe, bit 1 is reserved and must currently be +zero, and bit 2 marks a discontinuity. Codec initialization bytes are carried +reliably in `MediaTrackDescriptor`. QUIC already authenticates each DATAGRAM, +so the application header has no redundant checksum. + +Receivers must key reassembly by `(session_epoch, track_id, frame_sequence)`, +drop incomplete frames at their playback deadline, and reject fragments whose +offset plus payload exceeds `frame_size`. A new session epoch invalidates all +fragments retained from a previous connection. `frame_sequence` is generated +by the QUIC edge service and remains strictly increasing per track throughout +the media epoch, including after a device source restarts. diff --git a/protos/cmvr/quic_edge/v1/quic_edge.proto b/protos/cmvr/quic_edge/v1/quic_edge.proto new file mode 100644 index 00000000..b4f03a89 --- /dev/null +++ b/protos/cmvr/quic_edge/v1/quic_edge.proto @@ -0,0 +1,152 @@ +syntax = "proto3"; + +package cmvr.quic_edge.v1; + +// Every application message on the edge-opened reliable bidirectional stream +// is carried by this envelope. message_sequence is strictly increasing per +// sender (gaps are allowed) and is independent from the heartbeat sequence used +// for liveness acknowledgement. +message EdgeControlEnvelope { + uint32 protocol_version = 1; + uint64 message_sequence = 2; + + oneof payload { + NodeRegisterRequest node_register_request = 10; + NodeRegisterResponse node_register_response = 11; + NodeHeartbeat node_heartbeat = 12; + NodeHeartbeatAck node_heartbeat_ack = 13; + + MediaSessionOpen media_session_open = 20; + MediaTrackDescriptor media_track_descriptor = 21; + MediaSessionClose media_session_close = 22; + + ProtocolError protocol_error = 30; + } +} + +message NetworkInterfaceAddress { + enum AddressFamily { + ADDRESS_FAMILY_UNSPECIFIED = 0; + ADDRESS_FAMILY_IPV4 = 1; + ADDRESS_FAMILY_IPV6 = 2; + } + + string interface_name = 1; + string ip_address = 2; + AddressFamily family = 3; + bool loopback = 4; +} + +// The existing cmvr-es gRPC server remains the robot-control endpoint. The +// edge advertises its current reachable address through the QUIC control plane. +message GrpcEndpoint { + string host = 1; + uint32 port = 2; + bool tls = 3; +} + +message NodeDescriptor { + string node_id = 1; + string boot_id = 2; + string software_version = 3; + repeated NetworkInterfaceAddress local_interfaces = 4; + GrpcEndpoint grpc_endpoint = 5; +} + +// This must be the first application message sent after each QUIC connection +// is established. A reconnect always creates a new registration session. +message NodeRegisterRequest { + NodeDescriptor node = 1; + uint64 sent_at_unix_ms = 2; +} + +message NodeRegisterResponse { + bool accepted = 1; + string session_id = 2; + string message = 3; + + // Zero tells the edge to keep its locally configured interval. + uint32 heartbeat_interval_ms = 4; + + // Derived by the receiver from the authenticated QUIC peer address. It is + // not copied from a client-supplied local interface. + string observed_source_ip = 5; +} + +// A heartbeat carries a fresh network snapshot so address changes are reported +// without opening a second protocol or connection. +message NodeHeartbeat { + string node_id = 1; + string boot_id = 2; + string session_id = 3; + uint64 sequence = 4; + uint64 sent_at_unix_ms = 5; + string software_version = 6; + repeated NetworkInterfaceAddress local_interfaces = 7; + GrpcEndpoint grpc_endpoint = 8; +} + +message NodeHeartbeatAck { + bool accepted = 1; + uint64 acknowledged_sequence = 2; + string message = 3; + uint64 server_time_unix_ms = 4; + string observed_source_ip = 5; + string session_id = 6; +} + +message MediaSessionOpen { + string node_id = 1; + uint64 session_epoch = 2; + + // Binds the media epoch to the accepted node registration on this QUIC + // connection. Media must not start before this session is assigned. + string session_id = 3; +} + +enum MediaKind { + MEDIA_KIND_UNSPECIFIED = 0; + MEDIA_KIND_VIDEO = 1; + MEDIA_KIND_AUDIO = 2; +} + +message MediaTrackDescriptor { + uint32 track_id = 1; + MediaKind kind = 2; + string device_id = 3; + string codec = 4; + uint64 codec_generation = 5; + + // The exact MediaSourceHub track and the 32-bit token repeated in every + // DATAGRAM header. The full generation remains on the reliable stream. + string source_track_id = 6; + uint32 codec_generation_token = 7; + string payload_format = 8; + + // Video fields. They are zero for audio tracks. + uint32 width = 10; + uint32 height = 11; + uint32 frames_per_second = 12; + + // Audio fields. They are zero for video tracks. + uint32 sample_rate = 20; + uint32 channels = 21; + + // Decoder initialization bytes, for example AVCC/HVCC or AudioSpecificConfig. + // Existing cmvr-es sources may leave this empty when configuration NAL units + // are carried in-band. + bytes codec_config = 30; +} + +message MediaSessionClose { + string reason = 1; + string session_id = 2; + uint64 session_epoch = 3; +} + +message ProtocolError { + uint32 code = 1; + string message = 2; + uint64 related_message_sequence = 3; + bool fatal = 4; +}