diff --git a/cmvr-es/algorithms/controllers/CMakeLists.txt b/cmvr-es/algorithms/controllers/CMakeLists.txt index 1921842a..07fe6948 100644 --- a/cmvr-es/algorithms/controllers/CMakeLists.txt +++ b/cmvr-es/algorithms/controllers/CMakeLists.txt @@ -12,7 +12,7 @@ add_subdirectory(arm_control) #) 其他动态库类似 file(GLOB SRC ${CMAKE_CURRENT_SOURCE_DIR}/pid/src/pid_controller.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/ibvs/src/ibvs_controller.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/pbvs/src/pbvs_controller.cpp ) @@ -58,3 +58,13 @@ target_link_libraries(controller PUBLIC add_library(cmvr_es::algorithms::controller ALIAS controller) install(TARGETS controller LIBRARY DESTINATION lib) + +add_executable(pbvs_controller_test + pbvs/src/pbvs_controller_test.cpp +) + +target_link_libraries(pbvs_controller_test PRIVATE + cmvr_es::algorithms::controller + gtest + gtest_main +) diff --git a/cmvr-es/algorithms/controllers/pbvs/include/pbvs_controller.h b/cmvr-es/algorithms/controllers/pbvs/include/pbvs_controller.h new file mode 100644 index 00000000..645ddd83 --- /dev/null +++ b/cmvr-es/algorithms/controllers/pbvs/include/pbvs_controller.h @@ -0,0 +1,453 @@ +#pragma once + +#ifndef CMVR_PBVS_CONTROLLER_H +#define CMVR_PBVS_CONTROLLER_H + +#include + +#include + +#include "common/types/arm/arm_types.h" + +namespace cmvr { + +/** + * @brief 基于相对 3D 位姿的 PBVS 控制器。 + * + * 坐标系: + * + * G : Screen Tag 坐标系,同时作为屏幕参考坐标系 + * P : TCP / 触控点坐标系 + * + * 输入: + * + * ^G T_P_des : TCP 相对于 Screen Tag 的目标位姿 + * ^G T_P_cur : TCP 相对于 Screen Tag 的当前位姿 + * + * 输出: + * + * ^G V_P = + * + * [ vx ] + * [ vy ] + * [ vz ] + * [ wx ] + * [ wy ] + * [ wz ] + * + * 即 TCP 在 Screen Tag 坐标系 G 下表达的 6D Cartesian Twist。 + * + * 位置误差: + * + * e_p = p_des - p_cur + * + * 姿态误差: + * + * R_err = R_des * R_cur^T + * + * e_R = Log(R_err)^vee + * + * 控制律: + * + * v = Kp * e_p + * w = Kr * e_R + * + * 本类只负责视觉伺服 6D Twist 计算,不负责: + * + * - AprilTag 检测 + * - RobotArm + * - G -> Base 坐标转换 + * - IK + * - speedL 下发 + */ +class PbvsController { +public: + enum class ComputeStatus { + OK = 0, + TARGET_NOT_SET, + INVALID_INPUT, + INVALID_DT, + INVALID_TARGET, + INVALID_CURRENT_POSE + }; + + struct Output { + // 当前位置误差,单位 m + Eigen::Vector3d position_error_G{ + Eigen::Vector3d::Zero() + }; + + // 当前姿态误差 rotation-vector,单位 rad + Eigen::Vector3d rotation_error_G{ + Eigen::Vector3d::Zero() + }; + + // PBVS 计算得到的原始线速度,单位 m/s + Eigen::Vector3d raw_linear_velocity_G{ + Eigen::Vector3d::Zero() + }; + + // PBVS 计算得到的原始角速度,单位 rad/s + Eigen::Vector3d raw_angular_velocity_G{ + Eigen::Vector3d::Zero() + }; + + // 经过限幅 / 加速度 / 滤波后的线速度 + Eigen::Vector3d linear_velocity_G{ + Eigen::Vector3d::Zero() + }; + + // 经过限幅 / 加速度 / 滤波后的角速度 + Eigen::Vector3d angular_velocity_G{ + Eigen::Vector3d::Zero() + }; + + // 可直接取出的 CartesianVelocity。 + // + // 注意: + // 这里仍然是在 G / ScreenTag frame 下表达。 + device::CartesianVelocity twist_G{}; + + bool position_reached{false}; + bool orientation_reached{false}; + bool reached{false}; + + bool valid{false}; + }; + +public: + PbvsController(); + + /** + * @brief 设置完整目标位姿。 + * + * @param T_G_P_des TCP(P) 相对于 ScreenTag(G) 的目标位姿。 + */ + bool setTargetPose( + const Eigen::Matrix4d& T_G_P_des); + + /** + * @brief 使用目标位置 + 目标姿态设置期望位姿。 + */ + bool setTargetPose( + const Eigen::Vector3d& position_G, + const Eigen::Matrix3d& rotation_G_P); + + /** + * @brief 当前是否已经设置有效目标。 + */ + bool hasTarget() const { + return target_valid_; + } + + /** + * @brief 获取目标位姿。 + */ + const Eigen::Matrix4d& targetPose() const { + return T_G_P_des_; + } + + /** + * @brief 根据当前 TCP 位姿计算 PBVS 速度命令。 + * + * @param T_G_P_cur 当前 TCP 相对于 Screen Tag 的位姿。 + * @param dt 控制周期,单位 s。 + * @param output 输出结果。 + * + * @return 成功返回 true。 + */ + bool compute( + const Eigen::Matrix4d& T_G_P_cur, + double dt, + Output& output); + + /** + * @brief 便捷接口,只输出 CartesianVelocity。 + * + * 注意输出仍然是 G frame。 + */ + bool compute( + const Eigen::Matrix4d& T_G_P_cur, + double dt, + device::CartesianVelocity& twist_G_out); + + /** + * @brief 设置位置比例增益。 + * + * x/y/z 分别控制。 + */ + void setPositionGain( + const Eigen::Vector3d& kp); + + /** + * @brief 设置姿态比例增益。 + * + * rx/ry/rz 分别控制。 + */ + void setRotationGain( + const Eigen::Vector3d& kr); + + /** + * @brief 设置六维速度上限。 + * + * [vx vy vz wx wy wz] + * + * 前三维 m/s; + * 后三维 rad/s。 + */ + void setVelocityLimit6( + const std::array& vmax6); + + /** + * @brief 设置六维加速度限制。 + * + * 前三维 m/s^2; + * 后三维 rad/s^2。 + * + * <= 0 表示对应维度不限制。 + */ + void setAccelerationLimit6( + const std::array& amax6); + + /** + * @brief 设置六维误差阈值。 + * + * [x y z rx ry rz] + * + * 前三维 m; + * 后三维 rad。 + */ + void setTolerance6( + const std::array& tolerance6); + + /** + * @brief 设置一阶低通 alpha。 + * + * alpha = 1: + * 不滤波。 + * + * 0 < alpha < 1: + * + * cmd = + * alpha * current + * + + * (1-alpha) * previous + */ + void setTwistFilterAlpha(double alpha); + + /** + * @brief 是否启用每个轴。 + * + * 默认 6DoF 全部开启。 + * + * 例如以后如果不希望控制 yaw: + * + * enabled[5] = false; + */ + void setAxisEnabled( + const std::array& enabled); + + /** + * @brief 清空速度历史,但保留目标。 + */ + void resetTwistCommandState(); + + /** + * @brief 清空整个 PBVS 状态和目标。 + */ + void reset(); + + ComputeStatus lastComputeStatus() const { + return last_compute_status_; + } + + static const char* statusToString( + ComputeStatus status); + + const Eigen::Vector3d& lastPositionError() const { + return last_position_error_G_; + } + + const Eigen::Vector3d& lastRotationError() const { + return last_rotation_error_G_; + } + + const Eigen::Vector3d& positionGain() const { + return kp_position_; + } + + const Eigen::Vector3d& rotationGain() const { + return kp_rotation_; + } + + const std::array& velocityLimit6() const { + return vmax6_; + } + + const std::array& accelerationLimit6() const { + return amax6_; + } + + const std::array& tolerance6() const { + return tolerance6_; + } + + double twistFilterAlpha() const { + return twist_lpf_alpha_; + } + + const Eigen::Matrix& + lastTwistCommandG() const { + return last_twist_cmd_G_; + } + + bool lastReached() const { + return last_reached_; + } + +private: + static bool isFiniteTransform( + const Eigen::Matrix4d& T); + + static bool hasValidBottomRow( + const Eigen::Matrix4d& T, + double tolerance = 1e-6); + + static bool isValidTransform( + const Eigen::Matrix4d& T); + + /** + * @brief 将可能有微小数值误差的旋转矩阵投影到 SO(3)。 + */ + static Eigen::Matrix3d projectToSO3( + const Eigen::Matrix3d& R); + + /** + * @brief 计算空间旋转误差,在 G frame 表达。 + * + * R_err = R_des * R_cur^T + * + * e_R = Log(R_err)^vee + */ + static Eigen::Vector3d rotationError( + const Eigen::Matrix3d& R_des, + const Eigen::Matrix3d& R_cur); + + static double clampValue( + double value, + double lower, + double upper); + + static device::CartesianVelocity + toCartesianVelocity( + const Eigen::Matrix& twist); + +private: + // ---------------- target ---------------- + + Eigen::Matrix4d T_G_P_des_{ + Eigen::Matrix4d::Identity() + }; + + bool target_valid_{false}; + + // ---------------- gains ---------------- + + Eigen::Vector3d kp_position_{ + 2.0, + 2.0, + 1.5 + }; + + Eigen::Vector3d kp_rotation_{ + 1.5, + 1.5, + 1.5 + }; + + // ---------------- velocity limits ---------------- + + // + // [vx vy vz wx wy wz] + // + std::array vmax6_{{ + 0.10, + 0.10, + 0.05, + 0.50, + 0.50, + 0.50 + }}; + + // ---------------- acceleration limits ---------------- + + std::array amax6_{{ + 0.50, + 0.50, + 0.30, + 2.0, + 2.0, + 2.0 + }}; + + // ---------------- tolerance ---------------- + + std::array tolerance6_{{ + 0.0015, // x 1.5 mm + 0.0015, // y 1.5 mm + 0.0020, // z 2.0 mm + + 0.05, // rx ~2.9 deg + 0.05, // ry + 0.05 // rz + }}; + + // ---------------- axis enable ---------------- + + std::array axis_enabled_{{ + true, + true, + true, + true, + true, + true + }}; + + // ---------------- LPF ---------------- + + double twist_lpf_alpha_{1.0}; + + // ---------------- command history ---------------- + + Eigen::Matrix + previous_twist_cmd_G_{ + Eigen::Matrix::Zero() + }; + + bool has_previous_twist_{false}; + + // ---------------- last output ---------------- + + Eigen::Vector3d last_position_error_G_{ + Eigen::Vector3d::Zero() + }; + + Eigen::Vector3d last_rotation_error_G_{ + Eigen::Vector3d::Zero() + }; + + Eigen::Matrix + last_twist_cmd_G_{ + Eigen::Matrix::Zero() + }; + + bool last_reached_{false}; + + ComputeStatus last_compute_status_{ + ComputeStatus::TARGET_NOT_SET + }; +}; + +} // namespace cmvr + +#endif // CMVR_PBVS_CONTROLLER_H diff --git a/cmvr-es/algorithms/controllers/pbvs/src/pbvs_controller.cpp b/cmvr-es/algorithms/controllers/pbvs/src/pbvs_controller.cpp new file mode 100644 index 00000000..fedd1f87 --- /dev/null +++ b/cmvr-es/algorithms/controllers/pbvs/src/pbvs_controller.cpp @@ -0,0 +1,774 @@ +#include "algorithms/controllers/pbvs/include/pbvs_controller.h" + +#include +#include + +#include +#include + +namespace cmvr { + +PbvsController::PbvsController() +{ + resetTwistCommandState(); +} + +bool PbvsController::setTargetPose( + const Eigen::Matrix4d& T_G_P_des) +{ + if (!isValidTransform(T_G_P_des)) { + target_valid_ = false; + last_compute_status_ = + ComputeStatus::INVALID_TARGET; + return false; + } + + T_G_P_des_ = T_G_P_des; + + // + // 视觉位姿可能有微小数值误差。 + // 强制把 rotation 投影到 SO(3)。 + // + T_G_P_des_.block<3, 3>(0, 0) = + projectToSO3( + T_G_P_des.block<3, 3>(0, 0)); + + target_valid_ = true; + last_compute_status_ = + ComputeStatus::OK; + + return true; +} + +bool PbvsController::setTargetPose( + const Eigen::Vector3d& position_G, + const Eigen::Matrix3d& rotation_G_P) +{ + if (!position_G.allFinite() || + !rotation_G_P.allFinite()) { + + target_valid_ = false; + last_compute_status_ = + ComputeStatus::INVALID_TARGET; + + return false; + } + + Eigen::Matrix4d T = + Eigen::Matrix4d::Identity(); + + T.block<3, 3>(0, 0) = + projectToSO3(rotation_G_P); + + T.block<3, 1>(0, 3) = + position_G; + + return setTargetPose(T); +} + +bool PbvsController::compute( + const Eigen::Matrix4d& T_G_P_cur, + const double dt, + Output& output) +{ + output = Output{}; + + last_reached_ = false; + last_position_error_G_.setZero(); + last_rotation_error_G_.setZero(); + last_twist_cmd_G_.setZero(); + + if (!target_valid_) { + last_compute_status_ = + ComputeStatus::TARGET_NOT_SET; + + return false; + } + + if (!std::isfinite(dt) || + dt <= 0.0) { + + last_compute_status_ = + ComputeStatus::INVALID_DT; + + return false; + } + + if (!isValidTransform(T_G_P_cur)) { + last_compute_status_ = + ComputeStatus::INVALID_CURRENT_POSE; + + return false; + } + + // ===================================================== + // 1. Current pose + // ===================================================== + + const Eigen::Vector3d p_cur_G = + T_G_P_cur.block<3, 1>(0, 3); + + const Eigen::Matrix3d R_cur_G_P = + projectToSO3( + T_G_P_cur.block<3, 3>(0, 0)); + + // ===================================================== + // 2. Desired pose + // ===================================================== + + const Eigen::Vector3d p_des_G = + T_G_P_des_.block<3, 1>(0, 3); + + const Eigen::Matrix3d R_des_G_P = + T_G_P_des_.block<3, 3>(0, 0); + + // ===================================================== + // 3. Position error + // + // e_p = p_des - p_cur + // + // expressed in G frame. + // ===================================================== + + Eigen::Vector3d e_pos_G = + p_des_G - + p_cur_G; + + // ===================================================== + // 4. Rotation error + // + // R_err = + // R_des * R_cur^T + // + // e_R = + // Log(R_err)^vee + // + // e_R is also expressed in G frame. + // ===================================================== + + Eigen::Vector3d e_rot_G = + rotationError( + R_des_G_P, + R_cur_G_P); + + if (!e_pos_G.allFinite() || + !e_rot_G.allFinite()) { + + last_compute_status_ = + ComputeStatus::INVALID_INPUT; + + return false; + } + + // ===================================================== + // 5. Disabled axes + // ===================================================== + + for (int i = 0; i < 3; ++i) { + if (!axis_enabled_[i]) { + e_pos_G[i] = 0.0; + } + + if (!axis_enabled_[i + 3]) { + e_rot_G[i] = 0.0; + } + } + + last_position_error_G_ = + e_pos_G; + + last_rotation_error_G_ = + e_rot_G; + + output.position_error_G = + e_pos_G; + + output.rotation_error_G = + e_rot_G; + + // ===================================================== + // 6. Reached check + // ===================================================== + + bool position_reached = true; + bool orientation_reached = true; + + for (int i = 0; i < 3; ++i) { + + if (axis_enabled_[i] && + std::abs(e_pos_G[i]) > + tolerance6_[i]) { + + position_reached = false; + } + + if (axis_enabled_[i + 3] && + std::abs(e_rot_G[i]) > + tolerance6_[i + 3]) { + + orientation_reached = false; + } + } + + const bool reached = + position_reached && + orientation_reached; + + output.position_reached = + position_reached; + + output.orientation_reached = + orientation_reached; + + output.reached = + reached; + + last_reached_ = + reached; + + // ===================================================== + // 7. PBVS P control + // + // v = Kp * e_pos + // + // w = Kr * e_rot + // ===================================================== + + Eigen::Matrix + twist_raw_G = + Eigen::Matrix::Zero(); + + twist_raw_G.head<3>() = + kp_position_.cwiseProduct( + e_pos_G); + + twist_raw_G.tail<3>() = + kp_rotation_.cwiseProduct( + e_rot_G); + + // ===================================================== + // 8. Dead zone + // + // 某个轴已经进入误差阈值,则这个轴不再主动运动。 + // ===================================================== + + for (int i = 0; i < 6; ++i) { + + if (!axis_enabled_[i]) { + twist_raw_G[i] = 0.0; + continue; + } + + const double error_value = + i < 3 + ? e_pos_G[i] + : e_rot_G[i - 3]; + + if (std::abs(error_value) <= + tolerance6_[i]) { + + twist_raw_G[i] = 0.0; + } + } + + output.raw_linear_velocity_G = + twist_raw_G.head<3>(); + + output.raw_angular_velocity_G = + twist_raw_G.tail<3>(); + + // ===================================================== + // 9. Velocity limits + // ===================================================== + + Eigen::Matrix + twist_vel_limited = + twist_raw_G; + + for (int i = 0; i < 6; ++i) { + + const double vmax = + vmax6_[i]; + + if (!std::isfinite(vmax) || + vmax <= 0.0) { + + twist_vel_limited[i] = 0.0; + continue; + } + + twist_vel_limited[i] = + clampValue( + twist_vel_limited[i], + -vmax, + vmax); + } + + // ===================================================== + // 10. Reached: + // + // 进入完整目标阈值后直接输出 0。 + // + // 上层可以随后调用 RobotArm::stopL()。 + // ===================================================== + + if (reached) { + + previous_twist_cmd_G_.setZero(); + has_previous_twist_ = true; + + last_twist_cmd_G_.setZero(); + + output.linear_velocity_G.setZero(); + output.angular_velocity_G.setZero(); + + output.twist_G = + device::CartesianVelocity{}; + + output.valid = true; + + last_compute_status_ = + ComputeStatus::OK; + + return true; + } + + // ===================================================== + // 11. Acceleration limits + // ===================================================== + + Eigen::Matrix + twist_acc_limited; + + if (!has_previous_twist_) { + + previous_twist_cmd_G_.setZero(); + has_previous_twist_ = true; + } + + twist_acc_limited = + previous_twist_cmd_G_; + + for (int i = 0; i < 6; ++i) { + + if (!axis_enabled_[i]) { + twist_acc_limited[i] = 0.0; + continue; + } + + const double amax = + amax6_[i]; + + // <=0:不做加速度限制 + if (!std::isfinite(amax) || + amax <= 0.0) { + + twist_acc_limited[i] = + twist_vel_limited[i]; + + continue; + } + + const double dv_max = + amax * dt; + + const double dv_des = + twist_vel_limited[i] + - + previous_twist_cmd_G_[i]; + + const double dv = + clampValue( + dv_des, + -dv_max, + dv_max); + + twist_acc_limited[i] = + previous_twist_cmd_G_[i] + + + dv; + } + + // ===================================================== + // 12. First-order LPF + // ===================================================== + + Eigen::Matrix + twist_filtered = + twist_acc_limited; + + const double alpha = + std::clamp( + twist_lpf_alpha_, + 0.0, + 1.0); + + if (alpha > 0.0 && + alpha < 1.0) { + + twist_filtered = + alpha * + twist_acc_limited + + + (1.0 - alpha) * + previous_twist_cmd_G_; + } + + // ===================================================== + // 13. Make sure disabled axis is zero + // ===================================================== + + for (int i = 0; i < 6; ++i) { + if (!axis_enabled_[i]) { + twist_filtered[i] = 0.0; + } + } + + // ===================================================== + // 14. Save state + // ===================================================== + + previous_twist_cmd_G_ = + twist_filtered; + + last_twist_cmd_G_ = + twist_filtered; + + // ===================================================== + // 15. Output + // ===================================================== + + output.linear_velocity_G = + twist_filtered.head<3>(); + + output.angular_velocity_G = + twist_filtered.tail<3>(); + + output.twist_G = + toCartesianVelocity( + twist_filtered); + + output.valid = true; + + last_compute_status_ = + ComputeStatus::OK; + + return true; +} + +bool PbvsController::compute( + const Eigen::Matrix4d& T_G_P_cur, + const double dt, + device::CartesianVelocity& twist_G_out) +{ + Output output; + + if (!compute( + T_G_P_cur, + dt, + output)) { + + twist_G_out = + device::CartesianVelocity{}; + + return false; + } + + twist_G_out = + output.twist_G; + + return true; +} + +void PbvsController::setPositionGain( + const Eigen::Vector3d& kp) +{ + for (int i = 0; i < 3; ++i) { + if (std::isfinite(kp[i]) && + kp[i] >= 0.0) { + + kp_position_[i] = + kp[i]; + } + } +} + +void PbvsController::setRotationGain( + const Eigen::Vector3d& kr) +{ + for (int i = 0; i < 3; ++i) { + if (std::isfinite(kr[i]) && + kr[i] >= 0.0) { + + kp_rotation_[i] = + kr[i]; + } + } +} + +void PbvsController::setVelocityLimit6( + const std::array& vmax6) +{ + for (int i = 0; i < 6; ++i) { + + if (std::isfinite(vmax6[i]) && + vmax6[i] >= 0.0) { + + vmax6_[i] = + vmax6[i]; + } + } +} + +void PbvsController::setAccelerationLimit6( + const std::array& amax6) +{ + for (int i = 0; i < 6; ++i) { + + if (std::isfinite(amax6[i])) { + amax6_[i] = + amax6[i]; + } + } +} + +void PbvsController::setTolerance6( + const std::array& tolerance6) +{ + for (int i = 0; i < 6; ++i) { + + if (std::isfinite(tolerance6[i]) && + tolerance6[i] >= 0.0) { + + tolerance6_[i] = + tolerance6[i]; + } + } +} + +void PbvsController::setTwistFilterAlpha( + const double alpha) +{ + if (!std::isfinite(alpha)) { + return; + } + + twist_lpf_alpha_ = + std::clamp( + alpha, + 0.0, + 1.0); +} + +void PbvsController::setAxisEnabled( + const std::array& enabled) +{ + axis_enabled_ = + enabled; +} + +void PbvsController::resetTwistCommandState() +{ + previous_twist_cmd_G_.setZero(); + last_twist_cmd_G_.setZero(); + + has_previous_twist_ = false; +} + +void PbvsController::reset() +{ + T_G_P_des_.setIdentity(); + + target_valid_ = false; + + last_position_error_G_.setZero(); + last_rotation_error_G_.setZero(); + + last_reached_ = false; + + resetTwistCommandState(); + + last_compute_status_ = + ComputeStatus::TARGET_NOT_SET; +} + +const char* PbvsController::statusToString( + const ComputeStatus status) +{ + switch (status) { + + case ComputeStatus::OK: + return "ok"; + + case ComputeStatus::TARGET_NOT_SET: + return "target_not_set"; + + case ComputeStatus::INVALID_INPUT: + return "invalid_input"; + + case ComputeStatus::INVALID_DT: + return "invalid_dt"; + + case ComputeStatus::INVALID_TARGET: + return "invalid_target"; + + case ComputeStatus::INVALID_CURRENT_POSE: + return "invalid_current_pose"; + + default: + return "unknown"; + } +} + +bool PbvsController::isFiniteTransform( + const Eigen::Matrix4d& T) +{ + return T.allFinite(); +} + +bool PbvsController::hasValidBottomRow( + const Eigen::Matrix4d& T, + const double tolerance) +{ + return + std::abs(T(3, 0)) <= tolerance && + std::abs(T(3, 1)) <= tolerance && + std::abs(T(3, 2)) <= tolerance && + std::abs(T(3, 3) - 1.0) <= tolerance; +} + +bool PbvsController::isValidTransform( + const Eigen::Matrix4d& T) +{ + if (!isFiniteTransform(T)) { + return false; + } + + if (!hasValidBottomRow(T)) { + return false; + } + + return true; +} + +Eigen::Matrix3d PbvsController::projectToSO3( + const Eigen::Matrix3d& R) +{ + if (!R.allFinite()) { + return Eigen::Matrix3d::Identity(); + } + + Eigen::JacobiSVD svd( + R, + Eigen::ComputeFullU | + Eigen::ComputeFullV); + + Eigen::Matrix3d U = + svd.matrixU(); + + const Eigen::Matrix3d V = + svd.matrixV(); + + Eigen::Matrix3d R_projected = + U * V.transpose(); + + // + // 确保 det = +1,而不是 reflection。 + // + if (R_projected.determinant() < 0.0) { + + U.col(2) *= -1.0; + + R_projected = + U * V.transpose(); + } + + return R_projected; +} + +Eigen::Vector3d PbvsController::rotationError( + const Eigen::Matrix3d& R_des, + const Eigen::Matrix3d& R_cur) +{ + const Eigen::Matrix3d R_d = + projectToSO3(R_des); + + const Eigen::Matrix3d R_c = + projectToSO3(R_cur); + + // + // 空间旋转误差,在 G frame 表达。 + // + // 当前为 I,目标绕 +Z 旋转 theta: + // + // R_err = R_des + // + // 得到 +theta Z, + // 因此角速度方向正确。 + // + const Eigen::Matrix3d R_err = + R_d * + R_c.transpose(); + + Eigen::AngleAxisd aa( + R_err); + + const double angle = + aa.angle(); + + if (!std::isfinite(angle) || + std::abs(angle) <= 1e-12) { + + return Eigen::Vector3d::Zero(); + } + + const Eigen::Vector3d axis = + aa.axis(); + + if (!axis.allFinite()) { + return Eigen::Vector3d::Zero(); + } + + return axis * angle; +} + +double PbvsController::clampValue( + const double value, + const double lower, + const double upper) +{ + return std::max( + lower, + std::min( + value, + upper)); +} + +device::CartesianVelocity +PbvsController::toCartesianVelocity( + const Eigen::Matrix& twist) +{ + device::CartesianVelocity velocity; + + velocity.vx = + twist[0]; + + velocity.vy = + twist[1]; + + velocity.vz = + twist[2]; + + velocity.wx = + twist[3]; + + velocity.wy = + twist[4]; + + velocity.wz = + twist[5]; + + return velocity; +} + +} // namespace cmvr \ No newline at end of file diff --git a/cmvr-es/algorithms/controllers/pbvs/src/pbvs_controller_test.cpp b/cmvr-es/algorithms/controllers/pbvs/src/pbvs_controller_test.cpp new file mode 100644 index 00000000..cde596f8 --- /dev/null +++ b/cmvr-es/algorithms/controllers/pbvs/src/pbvs_controller_test.cpp @@ -0,0 +1,61 @@ +#include "gtest/gtest.h" + +#include + +#include + +#include "algorithms/controllers/pbvs/include/pbvs_controller.h" + +namespace { + +TEST(PbvsControllerTest, AppliesConfigurationSetters) { + cmvr::PbvsController controller; + + const Eigen::Vector3d position_gain(1.0, 2.0, 3.0); + const Eigen::Vector3d rotation_gain(4.0, 5.0, 6.0); + const std::array vmax{{0.1, 0.2, 0.3, 0.4, 0.5, 0.6}}; + const std::array amax{{1.0, 2.0, 3.0, 4.0, 5.0, 6.0}}; + const std::array tolerance{{0.001, 0.002, 0.003, 0.01, 0.02, 0.03}}; + + controller.setPositionGain(position_gain); + controller.setRotationGain(rotation_gain); + controller.setVelocityLimit6(vmax); + controller.setAccelerationLimit6(amax); + controller.setTolerance6(tolerance); + controller.setTwistFilterAlpha(0.75); + + EXPECT_TRUE(controller.positionGain().isApprox(position_gain)); + EXPECT_TRUE(controller.rotationGain().isApprox(rotation_gain)); + EXPECT_EQ(controller.velocityLimit6(), vmax); + EXPECT_EQ(controller.accelerationLimit6(), amax); + EXPECT_EQ(controller.tolerance6(), tolerance); + EXPECT_DOUBLE_EQ(controller.twistFilterAlpha(), 0.75); +} + +TEST(PbvsControllerTest, CommandsTowardPositionAndRotationError) { + cmvr::PbvsController controller; + controller.setPositionGain(Eigen::Vector3d::Ones()); + controller.setRotationGain(Eigen::Vector3d::Ones()); + controller.setVelocityLimit6({{1.0, 1.0, 1.0, 1.0, 1.0, 1.0}}); + controller.setAccelerationLimit6({{0.0, 0.0, 0.0, 0.0, 0.0, 0.0}}); + controller.setTolerance6({{0.0, 0.0, 0.0, 0.0, 0.0, 0.0}}); + controller.setTwistFilterAlpha(1.0); + + Eigen::Matrix4d target = Eigen::Matrix4d::Identity(); + target.block<3, 3>(0, 0) = + Eigen::AngleAxisd(0.25, Eigen::Vector3d::UnitZ()).toRotationMatrix(); + target.block<3, 1>(0, 3) = Eigen::Vector3d(0.1, -0.2, 0.3); + ASSERT_TRUE(controller.setTargetPose(target)); + + cmvr::PbvsController::Output output; + ASSERT_TRUE(controller.compute(Eigen::Matrix4d::Identity(), 0.01, output)); + ASSERT_TRUE(output.valid); + EXPECT_GT(output.linear_velocity_G.x(), 0.0); + EXPECT_LT(output.linear_velocity_G.y(), 0.0); + EXPECT_GT(output.linear_velocity_G.z(), 0.0); + EXPECT_NEAR(output.angular_velocity_G.x(), 0.0, 1e-12); + EXPECT_NEAR(output.angular_velocity_G.y(), 0.0, 1e-12); + EXPECT_GT(output.angular_velocity_G.z(), 0.0); +} + +} // namespace diff --git a/cmvr-es/algorithms/perception/CMakeLists.txt b/cmvr-es/algorithms/perception/CMakeLists.txt index 98eddf21..d47a2906 100644 --- a/cmvr-es/algorithms/perception/CMakeLists.txt +++ b/cmvr-es/algorithms/perception/CMakeLists.txt @@ -4,6 +4,7 @@ find_package(OpenCV REQUIRED) add_library(perception SHARED apriltag/src/tag_relative_target_3d.cpp apriltag/src/apriltag_perception.cpp + apriltag/src/tag_relative_tcp_pose.cpp ) target_include_directories(perception PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/cmvr-es/algorithms/perception/apriltag/include/tag_relative_tcp_pose.h b/cmvr-es/algorithms/perception/apriltag/include/tag_relative_tcp_pose.h new file mode 100644 index 00000000..7e41574d --- /dev/null +++ b/cmvr-es/algorithms/perception/apriltag/include/tag_relative_tcp_pose.h @@ -0,0 +1,289 @@ +#pragma once + +#ifndef CMVR_PERCEPTION_TAG_RELATIVE_TCP_POSE_H +#define CMVR_PERCEPTION_TAG_RELATIVE_TCP_POSE_H + +#include + +#include + +#include "algorithms/perception/apriltag/include/apriltag_perception.h" + +namespace cmvr::perception { + +/** + * @brief 根据同一台相机同时观测到的 Screen Tag 和 Hand Tag, + * 计算 TCP 相对于 Screen Tag 的位姿。 + * + * 坐标系: + * + * C : 固定外部相机坐标系 + * G : Screen Tag 坐标系,同时作为屏幕参考坐标系 + * H : Hand Tag 坐标系 + * P : TCP / 触控点坐标系 + * + * 已知: + * + * T_C_G : Screen Tag -> Camera + * T_C_H : Hand Tag -> Camera + * T_H_P : TCP -> Hand Tag + * + * 其中 T_H_P 是外部传入的一次标定结果。 + * + * 计算: + * + * T_G_H = inverse(T_C_G) * T_C_H + * + * T_G_P = T_G_H * T_H_P + * + * 即: + * + * T_G_P = inverse(T_C_G) * T_C_H * T_H_P + * + * 最终: + * + * p_G_P = T_G_P.block<3, 1>(0, 3) + * + * 得到 TCP 原点在 Screen Tag 坐标系下的位置。 + * + * 注意: + * - 本类不主动抓相机图像; + * - 本类不主动调用 AprilTagPerception::update(); + * - 上层应保证当前 perception 缓存来自同一帧; + * - Screen Tag 和 Hand Tag 必须同时在当前帧可见。 + */ +class TagRelativeTcpPose { +public: + enum class Status { + OK = 0, + + NO_PERCEPTION, + + INVALID_SCREEN_TAG_ID, + + INVALID_HAND_TAG_ID, + + SAME_TAG_ID, + + SCREEN_TAG_NOT_FOUND, + + HAND_TAG_NOT_FOUND, + + INVALID_T_C_G, + + INVALID_T_C_H, + + INVALID_T_H_P, + + INVALID_T_G_H, + + INVALID_T_G_P, + + INVALID_TCP_POSITION + }; + +public: + explicit TagRelativeTcpPose( + const std::shared_ptr& perception = nullptr); + + /** + * @brief 设置 AprilTag 感知前端。 + * + * 本类只读取感知缓存,不主动 update。 + */ + void setPerception( + const std::shared_ptr& perception); + + const std::shared_ptr& perception() const { + return perception_; + } + + /** + * @brief 设置 Screen Tag ID。 + */ + void setScreenTagId(int id); + + /** + * @brief 设置 Hand Tag ID。 + */ + void setHandTagId(int id); + + int screenTagId() const { + return screen_tag_id_; + } + + int handTagId() const { + return hand_tag_id_; + } + + /** + * @brief 使用当前 AprilTagPerception 缓存计算 TCP 相对 Screen Tag 的位姿。 + * + * 核心公式: + * + * T_G_P = + * inverse(T_C_G) + * * T_C_H + * * T_H_P + * + * @param T_H_P + * TCP(P) 相对于 Hand Tag(H) 的固定齐次变换。 + * + * 坐标变换语义: + * + * p_H = T_H_P * p_P + * + * 即: + * + * ^H T_P + * + * @return 成功返回 true。 + */ + bool update( + const Eigen::Matrix4d& T_H_P); + + /** + * @brief 当前结果是否有效。 + */ + bool valid() const { + return valid_; + } + + /** + * @brief 最近一次 update() 的状态。 + */ + Status lastStatus() const { + return last_status_; + } + + static const char* statusToString(Status status); + + /** + * @brief 当前 Screen Tag 在 Camera 中的位姿。 + * + * ^C T_G + */ + const Eigen::Matrix4d& T_C_G() const { + return T_C_G_; + } + + /** + * @brief 当前 Hand Tag 在 Camera 中的位姿。 + * + * ^C T_H + */ + const Eigen::Matrix4d& T_C_H() const { + return T_C_H_; + } + + /** + * @brief Hand Tag 相对于 Screen Tag 的位姿。 + * + * ^G T_H + */ + const Eigen::Matrix4d& T_G_H() const { + return T_G_H_; + } + + /** + * @brief TCP 相对于 Screen Tag 的完整 6DoF 位姿。 + * + * ^G T_P + */ + const Eigen::Matrix4d& T_G_P() const { + return T_G_P_; + } + + /** + * @brief TCP 原点在 Screen Tag 坐标系中的位置。 + * + * P_P^G = + * + * [ x_P ] + * [ y_P ] + * [ z_P ] + */ + const Eigen::Vector3d& tcpPositionInScreenTag() const { + return p_G_P_; + } + + /** + * @brief TCP 相对于 Screen Tag 的旋转矩阵。 + * + * R_G_P + */ + Eigen::Matrix3d tcpRotationInScreenTag() const { + return T_G_P_.block<3, 3>(0, 0); + } + + /** + * @brief 清空当前结果。 + */ + void clear(); + +private: + /** + * @brief 检查 4x4 矩阵元素是否全部有限。 + */ + static bool isFiniteTransform( + const Eigen::Matrix4d& T); + + /** + * @brief 基础检查齐次矩阵最后一行。 + */ + static bool hasValidHomogeneousBottomRow( + const Eigen::Matrix4d& T, + double tolerance = 1e-6); + + /** + * @brief 判断一个矩阵是否可以作为基本齐次变换使用。 + * + * 当前只检查: + * - 所有元素 finite + * - 最后一行约等于 [0 0 0 1] + * + * 暂时不强制检查 rotation orthonormal, + * 避免视觉估计中的微小数值误差导致误判。 + */ + static bool isValidTransform( + const Eigen::Matrix4d& T); + +private: + std::shared_ptr perception_{nullptr}; + + int screen_tag_id_{-1}; + int hand_tag_id_{-1}; + + // 当前外部相机观测 + Eigen::Matrix4d T_C_G_{ + Eigen::Matrix4d::Identity() + }; + + Eigen::Matrix4d T_C_H_{ + Eigen::Matrix4d::Identity() + }; + + // 相对变换 + Eigen::Matrix4d T_G_H_{ + Eigen::Matrix4d::Identity() + }; + + Eigen::Matrix4d T_G_P_{ + Eigen::Matrix4d::Identity() + }; + + // TCP 原点在 Screen Tag 坐标系的位置 + Eigen::Vector3d p_G_P_{ + Eigen::Vector3d::Zero() + }; + + bool valid_{false}; + + Status last_status_{ + Status::NO_PERCEPTION + }; +}; + +} // namespace cmvr::perception + +#endif // CMVR_PERCEPTION_TAG_RELATIVE_TCP_POSE_H \ No newline at end of file diff --git a/cmvr-es/algorithms/perception/apriltag/src/tag_relative_tcp_pose.cpp b/cmvr-es/algorithms/perception/apriltag/src/tag_relative_tcp_pose.cpp new file mode 100644 index 00000000..25501d8d --- /dev/null +++ b/cmvr-es/algorithms/perception/apriltag/src/tag_relative_tcp_pose.cpp @@ -0,0 +1,315 @@ +#include "algorithms/perception/apriltag/include/tag_relative_tcp_pose.h" + +#include + +namespace cmvr::perception { + +TagRelativeTcpPose::TagRelativeTcpPose( + const std::shared_ptr& perception) + : perception_(perception) +{ +} + +void TagRelativeTcpPose::setPerception( + const std::shared_ptr& perception) +{ + perception_ = perception; + clear(); + + if (!perception_) { + last_status_ = Status::NO_PERCEPTION; + } +} + +void TagRelativeTcpPose::setScreenTagId( + const int id) +{ + screen_tag_id_ = id; + valid_ = false; +} + +void TagRelativeTcpPose::setHandTagId( + const int id) +{ + hand_tag_id_ = id; + valid_ = false; +} + +bool TagRelativeTcpPose::update( + const Eigen::Matrix4d& T_H_P) +{ + valid_ = false; + + /* + * 1. 检查 perception + */ + if (!perception_) { + last_status_ = Status::NO_PERCEPTION; + return false; + } + + /* + * 2. 检查 Tag ID + */ + if (screen_tag_id_ < 0) { + last_status_ = Status::INVALID_SCREEN_TAG_ID; + return false; + } + + if (hand_tag_id_ < 0) { + last_status_ = Status::INVALID_HAND_TAG_ID; + return false; + } + + if (screen_tag_id_ == hand_tag_id_) { + last_status_ = Status::SAME_TAG_ID; + return false; + } + + /* + * 3. 检查外部传入的: + * + * ^H T_P + * + * Hand Tag -> TCP 固定标定矩阵。 + */ + if (!isValidTransform(T_H_P)) { + last_status_ = Status::INVALID_T_H_P; + return false; + } + + /* + * 4. 从 AprilTagPerception 当前缓存取 Screen Tag。 + * + * AprilTagPerception 已经通过 ViSP / PnP 得到: + * + * ^C T_G + */ + const auto* screen_tag = + perception_->findTag(screen_tag_id_); + + if (!screen_tag) { + last_status_ = + Status::SCREEN_TAG_NOT_FOUND; + return false; + } + + /* + * 5. 取 Hand Tag: + * + * ^C T_H + */ + const auto* hand_tag = + perception_->findTag(hand_tag_id_); + + if (!hand_tag) { + last_status_ = + Status::HAND_TAG_NOT_FOUND; + return false; + } + + /* + * 6. 保存当前帧两个原始视觉变换。 + * + * AprilTagPerception::Tag::T_c_t + * + * 定义是: + * + * Tag -> Camera + * + * 因此: + * + * screen tag: + * + * ^C T_G + * + * hand tag: + * + * ^C T_H + */ + T_C_G_ = screen_tag->T_c_t; + T_C_H_ = hand_tag->T_c_t; + + if (!isValidTransform(T_C_G_)) { + last_status_ = Status::INVALID_T_C_G; + return false; + } + + if (!isValidTransform(T_C_H_)) { + last_status_ = Status::INVALID_T_C_H; + return false; + } + + /* + * 7. 求 Hand Tag 相对于 Screen Tag 的位姿。 + * + * 已知: + * + * ^C T_G + * ^C T_H + * + * 因此: + * + * ^G T_H + * + * = (^C T_G)^-1 * ^C T_H + */ + T_G_H_ = + T_C_G_.inverse() * + T_C_H_; + + if (!isValidTransform(T_G_H_)) { + last_status_ = Status::INVALID_T_G_H; + return false; + } + + /* + * 8. 求 TCP 相对于 Screen Tag 的位姿。 + * + * 已知: + * + * ^G T_H + * ^H T_P + * + * 因此: + * + * ^G T_P + * + * = ^G T_H * ^H T_P + * + * = (^C T_G)^-1 + * * ^C T_H + * * ^H T_P + */ + T_G_P_ = + T_G_H_ * + T_H_P; + + if (!isValidTransform(T_G_P_)) { + last_status_ = Status::INVALID_T_G_P; + return false; + } + + /* + * 9. 提取 TCP 原点在 Screen Tag + * 坐标系 G 下的位置。 + * + * P_P^G = + * + * [ x_P ] + * [ y_P ] + * [ z_P ] + * + */ + p_G_P_ = + T_G_P_.block<3, 1>(0, 3); + + if (!p_G_P_.allFinite()) { + last_status_ = + Status::INVALID_TCP_POSITION; + return false; + } + + /* + * 10. 当前帧结果有效。 + */ + valid_ = true; + last_status_ = Status::OK; + + return true; +} + +void TagRelativeTcpPose::clear() +{ + T_C_G_.setIdentity(); + T_C_H_.setIdentity(); + + T_G_H_.setIdentity(); + T_G_P_.setIdentity(); + + p_G_P_.setZero(); + + valid_ = false; +} + +const char* TagRelativeTcpPose::statusToString( + const Status status) +{ + switch (status) { + + case Status::OK: + return "ok"; + + case Status::NO_PERCEPTION: + return "no_perception"; + + case Status::INVALID_SCREEN_TAG_ID: + return "invalid_screen_tag_id"; + + case Status::INVALID_HAND_TAG_ID: + return "invalid_hand_tag_id"; + + case Status::SAME_TAG_ID: + return "same_tag_id"; + + case Status::SCREEN_TAG_NOT_FOUND: + return "screen_tag_not_found"; + + case Status::HAND_TAG_NOT_FOUND: + return "hand_tag_not_found"; + + case Status::INVALID_T_C_G: + return "invalid_T_C_G"; + + case Status::INVALID_T_C_H: + return "invalid_T_C_H"; + + case Status::INVALID_T_H_P: + return "invalid_T_H_P"; + + case Status::INVALID_T_G_H: + return "invalid_T_G_H"; + + case Status::INVALID_T_G_P: + return "invalid_T_G_P"; + + case Status::INVALID_TCP_POSITION: + return "invalid_tcp_position"; + + default: + return "unknown"; + } +} + +bool TagRelativeTcpPose::isFiniteTransform( + const Eigen::Matrix4d& T) +{ + return T.allFinite(); +} + +bool TagRelativeTcpPose::hasValidHomogeneousBottomRow( + const Eigen::Matrix4d& T, + const double tolerance) +{ + return + std::abs(T(3, 0)) <= tolerance && + std::abs(T(3, 1)) <= tolerance && + std::abs(T(3, 2)) <= tolerance && + std::abs(T(3, 3) - 1.0) <= tolerance; +} + +bool TagRelativeTcpPose::isValidTransform( + const Eigen::Matrix4d& T) +{ + if (!isFiniteTransform(T)) { + return false; + } + + if (!hasValidHomogeneousBottomRow(T)) { + return false; + } + + return true; +} + +} // namespace cmvr::perception \ No newline at end of file diff --git a/cmvr-es/config/devices/camera/camera.pb.txt b/cmvr-es/config/devices/camera/camera.pb.txt index a6ee2278..038da4b6 100644 --- a/cmvr-es/config/devices/camera/camera.pb.txt +++ b/cmvr-es/config/devices/camera/camera.pb.txt @@ -92,7 +92,7 @@ camera { } consume_new_frame_only: false viewer_pip { - enable: true + enable: false left: -10 bottom: 10 width: 320 @@ -101,6 +101,36 @@ camera { } } + cameras { + id: "mujoco_external_touch_cam" + mujoco { + world_id: "mujoco_world" + camera_name: "external_touch_cam" + render { + width: 1280 + height: 720 + fps: 30 + stream_mode: STREAM_MODE_RGBD + } + encoder { + width: 480 + height: 320 + fps: 30 + codec: "H264" + enable_stream_timestamp: true + buffer_size: 30 + } + consume_new_frame_only: false + viewer_pip { + enable: false + left: 10 + bottom: 10 + width: 320 + height: 180 + } + } + } + cameras { id: "left_eye_cam" uvc { diff --git a/cmvr-es/config/devices/mujoco/right_arm_eye_to_hand_world.pb.txt b/cmvr-es/config/devices/mujoco/right_arm_eye_to_hand_world.pb.txt new file mode 100644 index 00000000..230ad18b --- /dev/null +++ b/cmvr-es/config/devices/mujoco/right_arm_eye_to_hand_world.pb.txt @@ -0,0 +1,7 @@ +worlds { + id: "mujoco_world" + model_path: "model/xiaoyan_description/right_arm_eye_to_hand.xml" + timestep_s: 0.001 + realtime_factor: 1.0 + require_actuator: true +} diff --git a/cmvr-es/config/manager/device_manager.pb.txt b/cmvr-es/config/manager/device_manager.pb.txt index 8aeb0d25..02667acf 100644 --- a/cmvr-es/config/manager/device_manager.pb.txt +++ b/cmvr-es/config/manager/device_manager.pb.txt @@ -5,7 +5,7 @@ device_manager { devices { id: "mujoco_world" type: DEVICE_TYPE_MUJOCO_WORLD - config_file: "devices/mujoco/mujoco_world.pb.txt" + config_file: "devices/mujoco/right_arm_eye_to_hand_world.pb.txt" enable: true } @@ -37,6 +37,13 @@ device_manager { enable: true } + devices { + id: "mujoco_external_touch_cam" + type: DEVICE_TYPE_CAMERA + config_file: "devices/camera/camera.pb.txt" + enable: true + } + devices { id: "right_hand_cam" type: DEVICE_TYPE_CAMERA diff --git a/cmvr-es/config/tasks/touch_screen_task/touch_screen_task.pb.txt b/cmvr-es/config/tasks/touch_screen_task/touch_screen_task.pb.txt index a9e8ad72..5625012d 100644 --- a/cmvr-es/config/tasks/touch_screen_task/touch_screen_task.pb.txt +++ b/cmvr-es/config/tasks/touch_screen_task/touch_screen_task.pb.txt @@ -5,6 +5,7 @@ touch_screen_task { arm_id: "right_arm" dexhand_id: "paxini_tip_1" camera_id: "right_hand_cam" + external_camera_id: "cam5" } initialization { @@ -27,37 +28,27 @@ touch_screen_task { depth_policy: TOUCH_SCREEN_DEPTH_POLICY_NONE target_point_method: TOUCH_SCREEN_TARGET_POINT_METHOD_TAG_PLANE } + external_apriltag { + tag_size_m: 0.012 + hand_tag_id: 1 + t_h_p { + m00: 1.0 m01: 0.0 m02: 0.0 m03: 0.0 + m10: 0.0 m11: 1.0 m12: 0.0 m13: -0.03 + m20: 0.0 m21: 0.0 m22: 1.0 m23: 0.0 + m30: 0.0 m31: 0.0 m32: 0.0 m33: 1.0 + } + } } alignment { - ibvs { - camera_link: "R_CAM" - lambda: 0.4 - mu: 0.1 - qdot_max: 1.0 - vmax6 { x: 1.0 y: 1.0 z: 1.0 rx: 0.6 ry: 0.6 rz: 0.6 } - amax6 { x: 2.4 y: 2.4 z: 4.5 rx: 2.5 ry: 2.5 rz: 2.5 } + pbvs { + position_gain { x: 2.0 y: 2.0 z: 1.5 } + rotation_gain { x: 1.5 y: 1.5 z: 1.5 } + vmax6 { x: 0.10 y: 0.10 z: 0.05 rx: 0.50 ry: 0.50 rz: 0.50 } + amax6 { x: 0.50 y: 0.50 z: 0.30 rx: 2.0 ry: 2.0 rz: 2.0 } twist_filter_alpha: 1.0 - r_camera_to_visp { - m00: 1.0 m01: 0.0 m02: 0.0 - m10: 0.0 m11: 1.0 m12: 0.0 - m20: 0.0 m21: 0.0 m22: 1.0 - } - r_camera_to_urdf { - m00: 1.0 m01: 0.0 m02: 0.0 - m10: 0.0 m11: 1.0 m12: 0.0 - m20: 0.0 m21: 0.0 m22: 1.0 - } - control_joint_names: "R_SHOULDER_P" - control_joint_names: "R_SHOULDER_R" - control_joint_names: "R_SHOULDER_Y" - control_joint_names: "R_ELBOW_R" - control_joint_names: "R_WRIST_P" - control_joint_names: "R_WRIST_Y" - control_joint_names: "R_WRIST_R" } target { - position_in_camera { x: -0.001 y: 0.08 z: 0.15 } rotation_vector { x: 3.14159265358979323846 y: 0.0 z: 0.0 } mode: TOUCH_SCREEN_ALIGN_MODE_RX_RY_AND_POSITION } diff --git a/cmvr-es/config/tasks/touch_screen_task/touch_screen_task_mujoco.pb.txt b/cmvr-es/config/tasks/touch_screen_task/touch_screen_task_mujoco.pb.txt index 91714aaf..951a3287 100644 --- a/cmvr-es/config/tasks/touch_screen_task/touch_screen_task_mujoco.pb.txt +++ b/cmvr-es/config/tasks/touch_screen_task/touch_screen_task_mujoco.pb.txt @@ -5,6 +5,7 @@ touch_screen_task { arm_id: "mujoco_right_arm" dexhand_id: "mujoco_zero_touch_dexhand" camera_id: "mujoco_hand_cam" + external_camera_id: "mujoco_external_touch_cam" } initialization { @@ -23,42 +24,33 @@ touch_screen_task { perception { apriltag { - tag_size_m: 0.12 + tag_size_m: 0.03 depth_policy: TOUCH_SCREEN_DEPTH_POLICY_NONE target_point_method: TOUCH_SCREEN_TARGET_POINT_METHOD_TAG_PLANE } + external_apriltag { + tag_size_m: 0.03 + hand_tag_id: 0 + # Hand Tag H -> TCP P after flipping the tag to face external_touch_cam. + t_h_p { + m00: 1.0 m01: 0.0 m02: 0.0 m03: 0.0 + m10: 0.0 m11: 0.0 m12: -1.0 m13: 0.0 + m20: 0.0 m21: 1.0 m22: 0.0 m23: -0.03 + m30: 0.0 m31: 0.0 m32: 0.0 m33: 1.0 + } + } } alignment { - ibvs { - camera_link: "R_CAM" - lambda: 0.4 - mu: 0.1 - qdot_max: 0.8 - vmax6 { x: 1.0 y: 1.0 z: 1.0 rx: 0.6 ry: 0.6 rz: 0.6 } - amax6 { x: 2.4 y: 2.4 z: 4.5 rx: 2.5 ry: 2.5 rz: 2.5 } + pbvs { + position_gain { x: 2.0 y: 2.0 z: 1.5 } + rotation_gain { x: 1.5 y: 1.5 z: 1.5 } + vmax6 { x: 0.10 y: 0.10 z: 0.05 rx: 0.50 ry: 0.50 rz: 0.50 } + amax6 { x: 0.50 y: 0.50 z: 0.30 rx: 2.0 ry: 2.0 rz: 2.0 } twist_filter_alpha: 1.0 - r_camera_to_visp { - m00: 1.0 m01: 0.0 m02: 0.0 - m10: 0.0 m11: -1.0 m12: 0.0 - m20: 0.0 m21: 0.0 m22: -1.0 - } - r_camera_to_urdf { - m00: 1.0 m01: 0.0 m02: 0.0 - m10: 0.0 m11: -1.0 m12: 0.0 - m20: 0.0 m21: 0.0 m22: -1.0 - } - control_joint_names: "R_SHOULDER_P" - control_joint_names: "R_SHOULDER_R" - control_joint_names: "R_SHOULDER_Y" - control_joint_names: "R_ELBOW_R" - control_joint_names: "R_WRIST_P" - control_joint_names: "R_WRIST_Y" - control_joint_names: "R_WRIST_R" } target { - position_in_camera { x: 0.0 y: 0.0 z: 0.30 } - rotation_vector { x: 3.14159265358979323846 y: 0.0 z: 0.0 } + rotation_vector { x: 1.57 y: 0.0 z: 0.0 } mode: TOUCH_SCREEN_ALIGN_MODE_POSITION_ONLY } error_threshold { @@ -78,7 +70,7 @@ touch_screen_task { speed_l { twist_tool { x: 0.0 y: -0.04 z: 0.0 rx: 0.0 ry: 0.0 rz: 0.0 } acceleration: 6.0 - max_distance_m: 0.12 + max_distance_m: 0.02 } tactile { finger: TOUCH_SCREEN_FINGER_TYPE_INDEX diff --git a/cmvr-es/devices/camera/mujoco_camera/src/mujoco_camera.cpp b/cmvr-es/devices/camera/mujoco_camera/src/mujoco_camera.cpp index 631405b9..fc2d728d 100644 --- a/cmvr-es/devices/camera/mujoco_camera/src/mujoco_camera.cpp +++ b/cmvr-es/devices/camera/mujoco_camera/src/mujoco_camera.cpp @@ -19,6 +19,13 @@ constexpr int kDefaultWidth = 640; constexpr int kDefaultHeight = 480; constexpr int kMaxGeom = 100000; +// GLFW keeps process-global initialization state. MuJoCo cameras render on +// independent threads, so serialize the one-time init/window creation path. +std::mutex& glfwInitMutex() { + static std::mutex mutex; + return mutex; +} + int positiveOrDefault(const int value, const int fallback) { return value > 0 ? value : fallback; @@ -453,6 +460,7 @@ bool MujocoCamera::initOffscreen_() return true; } + std::lock_guard glfw_lock(glfwInitMutex()); if (!glfwInit()) { setError_("[MujocoCamera] glfwInit failed"); return false; diff --git a/cmvr-es/manager/device_manager/src/device_manager.cpp b/cmvr-es/manager/device_manager/src/device_manager.cpp index 32e8319d..d8d8ac0c 100644 --- a/cmvr-es/manager/device_manager/src/device_manager.cpp +++ b/cmvr-es/manager/device_manager/src/device_manager.cpp @@ -308,12 +308,14 @@ void DeviceManager::configure_mujoco_viewer_pip_() auto viewer = getDevice(viewer_id); if (viewer && viewer->setPiPCameraConfig(camera_config)) { - camera->setFetchRgbdFn([viewer](std::vector& rgb, - std::vector& depth, - int& width, - int& height, - uint64_t& frame_id) { - return viewer->getPiPCameraRGBD(rgb, depth, width, height, frame_id); + const std::string camera_name = camera_config.camera_name(); + camera->setFetchRgbdFn([viewer, camera_name](std::vector& rgb, + std::vector& depth, + int& width, + int& height, + uint64_t& frame_id) { + return viewer->getPiPCameraRGBD( + camera_name, rgb, depth, width, height, frame_id); }); break; } diff --git a/cmvr-es/simulate/mujoco/mujoco_viewer/CMakeLists.txt b/cmvr-es/simulate/mujoco/mujoco_viewer/CMakeLists.txt index 54ce1b93..2902d956 100644 --- a/cmvr-es/simulate/mujoco/mujoco_viewer/CMakeLists.txt +++ b/cmvr-es/simulate/mujoco/mujoco_viewer/CMakeLists.txt @@ -35,4 +35,9 @@ target_link_libraries(mujoco_viewer_test gtest_main cmvr_es::mujoco_viewer cmvr_es::mujoco_world + cmvr_es::device_manager + cmvr_es::device::camera + cmvr_es::device::motor_manager + cmvr_es::device::mujoco_motor_driver + cmvr_es::device::motor_robot_arm ) diff --git a/cmvr-es/simulate/mujoco/mujoco_viewer/include/mujoco_viewer.h b/cmvr-es/simulate/mujoco/mujoco_viewer/include/mujoco_viewer.h index 74a18731..2a36b4b0 100644 --- a/cmvr-es/simulate/mujoco/mujoco_viewer/include/mujoco_viewer.h +++ b/cmvr-es/simulate/mujoco/mujoco_viewer/include/mujoco_viewer.h @@ -5,8 +5,11 @@ #pragma once #include +#include +#include #include #include +#include #include #include @@ -52,6 +55,14 @@ namespace cmvr { int display_height, int render_width, int render_height); + // Append another fixed camera view to the same viewer window. + void addPiPCamera(const char *camera_name, + int left, + int bottom, + int display_width, + int display_height, + int render_width, + int render_height); void disablePiPCamera(); // 获取 PiP 相机 RGB+Depth(Depth 已线性化为米) // depth 可不取(传 nullptr 或者用 getPiPCameraRGB 旧接口) @@ -60,9 +71,16 @@ namespace cmvr { int &width, int &height, uint64_t &frame_id) const; + bool getPiPCameraRGBD(const std::string &camera_name, + std::vector &rgb, + std::vector &depth, + int &width, + int &height, + uint64_t &frame_id) const; // 只拿 frame_id,便于 physics 线程判断是否新帧 uint64_t getPiPCameraFrameId() const; + uint64_t getPiPCameraFrameId(const std::string &camera_name) const; public: void setupCamera(double distance = 3.0, @@ -79,9 +97,34 @@ namespace cmvr { void printCameraState() const; private: + struct PiPCameraState { + std::string name; + int camera_id = -1; + int width = 320; + int height = 240; + int render_width = 320; + int render_height = 240; + bool render_size_warning_logged = false; + int margin = 10; + bool custom_pos = false; + int left = 0; + int bottom = 0; + mjvCamera camera{}; + mjvScene scene{}; + bool scene_inited = false; + mjModel *scene_model = nullptr; + std::vector rgb; + std::vector depth; + int rgb_width = 0; + int rgb_height = 0; + bool rgb_valid = false; + uint64_t frame_id = 0; + }; + void renderPiP(); void initSim(); void syncThreadFunc(); + void clearPiPCameras(); private: std::shared_ptr world_; @@ -93,31 +136,10 @@ namespace cmvr { std::unique_ptr sim_; std::thread sync_thread_; - bool pip_enabled_ = false; - std::string pip_camera_name_; - int pip_camera_id_ = -1; - int pip_width_ = 320; - int pip_height_ = 240; - int pip_render_width_ = 320; - int pip_render_height_ = 240; - bool pip_render_size_warning_logged_ = false; - int pip_margin_ = 10; - bool pip_custom_pos_ = false; - int pip_left_ = 0; - int pip_bottom_ = 0; - mjvCamera pip_cam_; - mjvScene pip_scene_; - bool pip_scene_inited_ = false; - mjModel *pip_scene_model_ = nullptr; + std::deque pip_cameras_; mjData *pip_render_data_ = nullptr; mjModel *pip_render_data_model_ = nullptr; mutable std::mutex pip_rgb_mtx_; - std::vector pip_rgb_; - std::vector pip_depth_; // 新增:z-buffer - int pip_rgb_width_ = 0; - int pip_rgb_height_ = 0; - bool pip_rgb_valid_ = false; - uint64_t pip_frame_id_ = 0; // 新增:帧序号 }; @@ -139,15 +161,20 @@ namespace cmvr { int& width, int& height, uint64_t& frame_id) const; + bool getPiPCameraRGBD(const std::string& camera_name, + std::vector& rgb, + std::vector& depth, + int& width, + int& height, + uint64_t& frame_id) const; private: config::MujocoViewerConfig config_; - config::MujocoCameraConfig pip_camera_config_; + std::vector pip_camera_configs_; std::shared_ptr world_; std::unique_ptr viewer_; std::thread viewer_thread_; mutable std::mutex mtx_; - bool has_pip_camera_config_ = false; bool running_ = false; bool stop_requested_ = false; }; diff --git a/cmvr-es/simulate/mujoco/mujoco_viewer/src/mujoco_viewer.cpp b/cmvr-es/simulate/mujoco/mujoco_viewer/src/mujoco_viewer.cpp index a5d84cd5..236ebed5 100644 --- a/cmvr-es/simulate/mujoco/mujoco_viewer/src/mujoco_viewer.cpp +++ b/cmvr-es/simulate/mujoco/mujoco_viewer/src/mujoco_viewer.cpp @@ -53,8 +53,6 @@ namespace cmvr { mjv_defaultCamera(&cam_); mjv_defaultOption(&opt_); mjv_defaultPerturb(&pert_); - mjv_defaultCamera(&pip_cam_); - mjv_defaultScene(&pip_scene_); auto platform_ui = std::make_unique(this); sim_ = std::make_unique( @@ -72,10 +70,7 @@ namespace cmvr { sync_thread_.join(); } - if (pip_scene_inited_) { - mjv_freeScene(&pip_scene_); - pip_scene_inited_ = false; - } + clearPiPCameras(); if (pip_render_data_ != nullptr) { mj_deleteData(pip_render_data_); pip_render_data_ = nullptr; @@ -120,14 +115,12 @@ namespace cmvr { } void MuJocoViewer::enablePiPCamera(const char *camera_name) { - pip_enabled_ = true; - pip_camera_name_ = camera_name ? camera_name : ""; - pip_camera_id_ = -1; - pip_width_ = 320; - pip_height_ = 240; - pip_render_width_ = pip_width_; - pip_render_height_ = pip_height_; - pip_custom_pos_ = false; + clearPiPCameras(); + PiPCameraState state; + state.name = camera_name ? camera_name : ""; + mjv_defaultCamera(&state.camera); + mjv_defaultScene(&state.scene); + pip_cameras_.push_back(std::move(state)); } void MuJocoViewer::enablePiPCamera(const char *camera_name, @@ -145,77 +138,66 @@ namespace cmvr { int display_height, int render_width, int render_height) { - pip_enabled_ = true; - pip_camera_name_ = camera_name ? camera_name : ""; - pip_camera_id_ = -1; - pip_left_ = left; - pip_bottom_ = bottom; - pip_width_ = display_width > 0 ? display_width : 320; - pip_height_ = display_height > 0 ? display_height : 240; - pip_render_width_ = render_width > 0 ? render_width : pip_width_; - pip_render_height_ = render_height > 0 ? render_height : pip_height_; - pip_custom_pos_ = true; + clearPiPCameras(); + addPiPCamera(camera_name, + left, + bottom, + display_width, + display_height, + render_width, + render_height); + } + + void MuJocoViewer::addPiPCamera(const char *camera_name, + int left, + int bottom, + int display_width, + int display_height, + int render_width, + int render_height) { + PiPCameraState state; + state.name = camera_name ? camera_name : ""; + state.left = left; + state.bottom = bottom; + state.width = display_width > 0 ? display_width : 320; + state.height = display_height > 0 ? display_height : 240; + state.render_width = render_width > 0 ? render_width : state.width; + state.render_height = render_height > 0 ? render_height : state.height; + state.custom_pos = true; + mjv_defaultCamera(&state.camera); + mjv_defaultScene(&state.scene); + pip_cameras_.push_back(std::move(state)); } void MuJocoViewer::disablePiPCamera() { - pip_enabled_ = false; + clearPiPCameras(); + } + + void MuJocoViewer::clearPiPCameras() { + for (auto &pip : pip_cameras_) { + if (pip.scene_inited) { + mjv_freeScene(&pip.scene); + pip.scene_inited = false; + pip.scene_model = nullptr; + } + } + pip_cameras_.clear(); } void MuJocoViewer::renderPiP() { - if (!pip_enabled_ || !sim_) return; - if (pip_camera_name_.empty()) return; + if (!sim_ || pip_cameras_.empty()) return; mjModel* render_model = sim_->m_passive_ ? sim_->m_passive_ : sim_->m_; mjData* render_data = sim_->d_passive_ ? sim_->d_passive_ : sim_->d_; if (!render_model || !render_data) return; - if (pip_camera_id_ < 0) { - pip_camera_id_ = mj_name2id(render_model, mjOBJ_CAMERA, pip_camera_name_.c_str()); - if (pip_camera_id_ < 0) { - return; - } - } - auto [fb_width, fb_height] = sim_->platform_ui->GetFramebufferSize(); if (fb_width <= 0 || fb_height <= 0) return; - int left = 0; - int bottom = 0; - int width = 0; - int height = 0; - - if (pip_custom_pos_) { - width = std::min(pip_width_, fb_width); - height = std::min(pip_height_, fb_height); - const int max_left = fb_width - width; - const int max_bottom = fb_height - height; - left = pip_left_ >= 0 - ? std::max(0, std::min(pip_left_, max_left)) - : std::max(0, std::min(fb_width - width + pip_left_, max_left)); - bottom = pip_bottom_ >= 0 - ? std::max(0, std::min(pip_bottom_, max_bottom)) - : std::max(0, std::min(fb_height - height + pip_bottom_, max_bottom)); - } else { - width = std::min(pip_width_, fb_width - 2 * pip_margin_); - height = std::min(pip_height_, fb_height - 2 * pip_margin_); - left = fb_width - pip_margin_ - width; - bottom = pip_margin_; - } - - if (width <= 0 || height <= 0) return; - - mjrRect display_rect; - display_rect.width = width; - display_rect.height = height; - display_rect.left = left; - display_rect.bottom = bottom; - - // Copy only the visualization state while synchronized with Simulate. - // Keep scene update, GPU rendering and pixel readback out of this lock - // so the world sync thread cannot hold the simulation mutex while - // waiting for the viewer. + // Copy one visualization snapshot for all PiP cameras. Keep the + // simulation lock out of scene updates, GPU rendering, and readback. { std::unique_lock lock(sim_->mtx, std::try_to_lock); if (lock.owns_lock()) { @@ -233,121 +215,164 @@ namespace cmvr { } } - // The simulation lock is intentionally non-blocking. If the sync - // thread owns it, use the last complete snapshot so every back buffer - // still gets the PiP overlay; skip only until the first snapshot exists. + // If the sync thread owns the lock, use the last complete snapshot. if (pip_render_data_ == nullptr || pip_render_data_model_ != render_model) { return; } - if (!pip_scene_inited_ || pip_scene_model_ != render_model) { - if (pip_scene_inited_) { - mjv_freeScene(&pip_scene_); - } - mjv_makeScene(render_model, &pip_scene_, kPiPMaxGeom); - pip_scene_inited_ = true; - pip_scene_model_ = render_model; - } - - pip_cam_.type = mjCAMERA_FIXED; - pip_cam_.fixedcamid = pip_camera_id_; - pip_cam_.trackbodyid = -1; - - mjv_updateScene(render_model, pip_render_data_, &opt_, &pert_, &pip_cam_, mjCAT_ALL, &pip_scene_); - auto& context = sim_->platform_ui->mjr_context(); - const int offscreen_width = context.offWidth > 0 ? context.offWidth : display_rect.width; - const int offscreen_height = context.offHeight > 0 ? context.offHeight : display_rect.height; - const int render_width = std::min(std::max(1, pip_render_width_), offscreen_width); - const int render_height = std::min(std::max(1, pip_render_height_), offscreen_height); - if (!pip_render_size_warning_logged_ && - (render_width != pip_render_width_ || render_height != pip_render_height_)) { - CMVR_LOG(WARNING) << "[MuJocoViewer] PiP camera render size clamped" - << ", requested=" << pip_render_width_ << "x" << pip_render_height_ - << ", actual=" << render_width << "x" << render_height - << ", offscreen=" << offscreen_width << "x" << offscreen_height; - pip_render_size_warning_logged_ = true; - } - const bool use_offscreen = render_width != display_rect.width || render_height != display_rect.height; + for (size_t pip_index = 0; pip_index < pip_cameras_.size(); ++pip_index) { + auto& pip = pip_cameras_[pip_index]; + if (pip.name.empty()) continue; - mjrRect render_rect; - render_rect.left = 0; - render_rect.bottom = 0; - render_rect.width = render_width; - render_rect.height = render_height; + if (pip.camera_id < 0) { + pip.camera_id = mj_name2id(render_model, mjOBJ_CAMERA, pip.name.c_str()); + if (pip.camera_id < 0) { + CMVR_LOG(WARNING) << "[MuJocoViewer] PiP camera not found: " << pip.name; + continue; + } + } - bool rendered_offscreen = false; - if (use_offscreen) { - mjr_setBuffer(mjFB_OFFSCREEN, &context); - if (context.currentBuffer == mjFB_OFFSCREEN) { - mjr_render(render_rect, &pip_scene_, &context); - rendered_offscreen = true; + int left = 0; + int bottom = 0; + int width = 0; + int height = 0; + if (pip.custom_pos) { + width = std::min(pip.width, fb_width); + height = std::min(pip.height, fb_height); + const int max_left = fb_width - width; + const int max_bottom = fb_height - height; + left = pip.left >= 0 + ? std::max(0, std::min(pip.left, max_left)) + : std::max(0, std::min(fb_width - width + pip.left, max_left)); + bottom = pip.bottom >= 0 + ? std::max(0, std::min(pip.bottom, max_bottom)) + : std::max(0, std::min(fb_height - height + pip.bottom, max_bottom)); } else { - mjr_render(display_rect, &pip_scene_, &context); + width = std::min(pip.width, fb_width - 2 * pip.margin); + height = std::min(pip.height, fb_height - 2 * pip.margin); + left = fb_width - pip.margin - width; + bottom = pip.margin; + } + if (width <= 0 || height <= 0) continue; + + mjrRect display_rect; + display_rect.width = width; + display_rect.height = height; + display_rect.left = left; + display_rect.bottom = bottom; + + if (!pip.scene_inited || pip.scene_model != render_model) { + if (pip.scene_inited) { + mjv_freeScene(&pip.scene); + } + mjv_makeScene(render_model, &pip.scene, kPiPMaxGeom); + pip.scene_inited = true; + pip.scene_model = render_model; + } + + pip.camera.type = mjCAMERA_FIXED; + pip.camera.fixedcamid = pip.camera_id; + pip.camera.trackbodyid = -1; + mjv_updateScene(render_model, + pip_render_data_, + &opt_, + &pert_, + &pip.camera, + mjCAT_ALL, + &pip.scene); + + const int offscreen_width = context.offWidth > 0 ? context.offWidth : display_rect.width; + const int offscreen_height = context.offHeight > 0 ? context.offHeight : display_rect.height; + const int render_width = std::min(std::max(1, pip.render_width), offscreen_width); + const int render_height = std::min(std::max(1, pip.render_height), offscreen_height); + if (!pip.render_size_warning_logged && + (render_width != pip.render_width || render_height != pip.render_height)) { + CMVR_LOG(WARNING) << "[MuJocoViewer] PiP camera render size clamped" + << ", camera=" << pip.name + << ", requested=" << pip.render_width << "x" << pip.render_height + << ", actual=" << render_width << "x" << render_height + << ", offscreen=" << offscreen_width << "x" << offscreen_height; + pip.render_size_warning_logged = true; + } + const bool use_offscreen = + render_width != display_rect.width || render_height != display_rect.height; + + mjrRect render_rect; + render_rect.left = 0; + render_rect.bottom = 0; + render_rect.width = render_width; + render_rect.height = render_height; + + bool rendered_offscreen = false; + if (use_offscreen) { + mjr_setBuffer(mjFB_OFFSCREEN, &context); + if (context.currentBuffer == mjFB_OFFSCREEN) { + mjr_render(render_rect, &pip.scene, &context); + rendered_offscreen = true; + } else { + mjr_render(display_rect, &pip.scene, &context); + render_rect = display_rect; + } + } else { + mjr_render(display_rect, &pip.scene, &context); render_rect = display_rect; } - } else { - mjr_render(display_rect, &pip_scene_, &context); - render_rect = display_rect; - } - { - std::lock_guard lock(pip_rgb_mtx_); - const int w = render_rect.width; - const int h = render_rect.height; - if (w > 0 && h > 0) { - pip_rgb_.resize(static_cast(3 * w * h)); - pip_depth_.resize(static_cast(w * h)); + { + std::lock_guard lock(pip_rgb_mtx_); + const int w = render_rect.width; + const int h = render_rect.height; + if (w > 0 && h > 0) { + pip.rgb.resize(static_cast(3 * w * h)); + pip.depth.resize(static_cast(w * h)); + mjr_readPixels(pip.rgb.data(), pip.depth.data(), render_rect, &context); - // 同时读 RGB 和 depth(z-buffer 0..1) - mjr_readPixels(pip_rgb_.data(), pip_depth_.data(), - render_rect, &context); - if (rendered_offscreen) { - mjr_setBuffer(mjFB_WINDOW, &context); - mjr_render(display_rect, &pip_scene_, &context); - } + // OpenGL's pixel origin is bottom-left; expose top-left images. + for (int r = 0; r < h / 2; ++r) { + unsigned char *top_row = pip.rgb.data() + 3 * w * r; + unsigned char *bottom_row = pip.rgb.data() + 3 * w * (h - 1 - r); + std::swap_ranges(top_row, top_row + 3 * w, bottom_row); - // OpenGL 像素原点在左下,需要竖直翻转 RGB 和 depth - for (int r = 0; r < h / 2; ++r) { - // flip rgb row - unsigned char *top_row = pip_rgb_.data() + 3 * w * r; - unsigned char *bottom_row = pip_rgb_.data() + 3 * w * (h - 1 - r); - std::swap_ranges(top_row, top_row + 3 * w, bottom_row); - - // flip depth row - float *top_d = pip_depth_.data() + w * r; - float *bot_d = pip_depth_.data() + w * (h - 1 - r); - std::swap_ranges(top_d, top_d + w, bot_d); - } - - // 将 OpenGL depth buffer(0..1) 线性化为相机前向距离(米)。 - const double znear = static_cast(render_model->vis.map.znear) * - static_cast(render_model->stat.extent); - const double zfar = static_cast(render_model->vis.map.zfar) * - static_cast(render_model->stat.extent); - if (znear > 0.0 && zfar > znear) { - const double two_nf = 2.0 * znear * zfar; - const double f_plus_n = zfar + znear; - const double f_minus_n = zfar - znear; - for (float &d : pip_depth_) { - if (!std::isfinite(d) || d <= 0.0f || d >= 1.0f) { - d = std::numeric_limits::infinity(); - continue; - } - const double z_ndc = 2.0 * static_cast(d) - 1.0; // [-1,1] - const double denom = f_plus_n - z_ndc * f_minus_n; - if (denom <= 1e-12) { - d = std::numeric_limits::infinity(); - continue; - } - d = static_cast(two_nf / denom); + float *top_d = pip.depth.data() + w * r; + float *bot_d = pip.depth.data() + w * (h - 1 - r); + std::swap_ranges(top_d, top_d + w, bot_d); } - } - pip_rgb_width_ = w; - pip_rgb_height_ = h; - pip_rgb_valid_ = true; - ++pip_frame_id_; // 新帧 + // Linearize the OpenGL depth buffer to camera-forward meters. + const double znear = static_cast(render_model->vis.map.znear) * + static_cast(render_model->stat.extent); + const double zfar = static_cast(render_model->vis.map.zfar) * + static_cast(render_model->stat.extent); + if (znear > 0.0 && zfar > znear) { + const double two_nf = 2.0 * znear * zfar; + const double f_plus_n = zfar + znear; + const double f_minus_n = zfar - znear; + for (float &d : pip.depth) { + if (!std::isfinite(d) || d <= 0.0f || d >= 1.0f) { + d = std::numeric_limits::infinity(); + continue; + } + const double z_ndc = 2.0 * static_cast(d) - 1.0; + const double denom = f_plus_n - z_ndc * f_minus_n; + if (denom <= 1e-12) { + d = std::numeric_limits::infinity(); + continue; + } + d = static_cast(two_nf / denom); + } + } + + pip.rgb_width = w; + pip.rgb_height = h; + pip.rgb_valid = true; + ++pip.frame_id; + } + } + + if (rendered_offscreen) { + mjr_setBuffer(mjFB_WINDOW, &context); + mjr_render(display_rect, &pip.scene, &context); } } } @@ -362,12 +387,18 @@ namespace cmvr { if (!world_->model() || !world_->data()) { mju_error("MuJocoViewer world has null model/data"); } - if (pip_enabled_ && pip_render_width_ > 0 && pip_render_height_ > 0) { + int max_pip_render_width = 0; + int max_pip_render_height = 0; + for (const auto &pip : pip_cameras_) { + max_pip_render_width = std::max(max_pip_render_width, pip.render_width); + max_pip_render_height = std::max(max_pip_render_height, pip.render_height); + } + if (!pip_cameras_.empty() && max_pip_render_width > 0 && max_pip_render_height > 0) { mjModel* model = world_->model(); const int old_width = model->vis.global.offwidth; const int old_height = model->vis.global.offheight; - model->vis.global.offwidth = std::max(model->vis.global.offwidth, pip_render_width_); - model->vis.global.offheight = std::max(model->vis.global.offheight, pip_render_height_); + model->vis.global.offwidth = std::max(model->vis.global.offwidth, max_pip_render_width); + model->vis.global.offheight = std::max(model->vis.global.offheight, max_pip_render_height); if (model->vis.global.offwidth != old_width || model->vis.global.offheight != old_height) { CMVR_LOG(INFO) << "[MuJocoViewer] resize offscreen buffer before context creation" << ", old=" << old_width << "x" << old_height @@ -417,7 +448,15 @@ namespace cmvr { uint64_t MuJocoViewer::getPiPCameraFrameId() const { std::lock_guard lock(pip_rgb_mtx_); - return pip_frame_id_; + return pip_cameras_.empty() ? 0 : pip_cameras_.front().frame_id; + } + + uint64_t MuJocoViewer::getPiPCameraFrameId(const std::string &camera_name) const { + std::lock_guard lock(pip_rgb_mtx_); + const auto it = std::find_if( + pip_cameras_.begin(), pip_cameras_.end(), + [&camera_name](const PiPCameraState &pip) { return pip.name == camera_name; }); + return it == pip_cameras_.end() ? 0 : it->frame_id; } bool MuJocoViewer::getPiPCameraRGBD(std::vector &rgb, @@ -426,13 +465,35 @@ namespace cmvr { int &height, uint64_t &frame_id) const { std::lock_guard lock(pip_rgb_mtx_); - if (!pip_rgb_valid_ || pip_rgb_.empty()) return false; + if (pip_cameras_.empty()) return false; + const auto &pip = pip_cameras_.front(); + if (!pip.rgb_valid || pip.rgb.empty()) return false; - rgb = pip_rgb_; - depth = pip_depth_; - width = pip_rgb_width_; - height = pip_rgb_height_; - frame_id = pip_frame_id_; + rgb = pip.rgb; + depth = pip.depth; + width = pip.rgb_width; + height = pip.rgb_height; + frame_id = pip.frame_id; + return true; + } + + bool MuJocoViewer::getPiPCameraRGBD(const std::string &camera_name, + std::vector &rgb, + std::vector &depth, + int &width, + int &height, + uint64_t &frame_id) const { + std::lock_guard lock(pip_rgb_mtx_); + const auto it = std::find_if( + pip_cameras_.begin(), pip_cameras_.end(), + [&camera_name](const PiPCameraState &pip) { return pip.name == camera_name; }); + if (it == pip_cameras_.end() || !it->rgb_valid || it->rgb.empty()) return false; + + rgb = it->rgb; + depth = it->depth; + width = it->rgb_width; + height = it->rgb_height; + frame_id = it->frame_id; return true; } @@ -487,6 +548,7 @@ namespace cmvr { } std::shared_ptr world; + std::vector pip_camera_configs; { std::lock_guard lock(mtx_); if (running_) { @@ -497,6 +559,7 @@ namespace cmvr { return false; } world = world_; + pip_camera_configs = pip_camera_configs_; stop_requested_ = false; running_ = true; } @@ -517,19 +580,30 @@ namespace cmvr { config_.camera_azimuth(), config_.camera_elevation()); - if (has_pip_camera_config_ && !pip_camera_config_.camera_name().empty()) { - const auto& pip = pip_camera_config_.viewer_pip(); - const auto& render = pip_camera_config_.render(); - if (pip.width() > 0 && pip.height() > 0) { - viewer->enablePiPCamera(pip_camera_config_.camera_name().c_str(), + bool first_pip_camera = true; + for (const auto& camera_config : pip_camera_configs) { + if (camera_config.camera_name().empty()) { + continue; + } + const auto& pip = camera_config.viewer_pip(); + const auto& render = camera_config.render(); + if (first_pip_camera) { + viewer->enablePiPCamera(camera_config.camera_name().c_str(), pip.left(), pip.bottom(), pip.width(), pip.height(), render.width(), render.height()); + first_pip_camera = false; } else { - viewer->enablePiPCamera(pip_camera_config_.camera_name().c_str()); + viewer->addPiPCamera(camera_config.camera_name().c_str(), + pip.left(), + pip.bottom(), + pip.width(), + pip.height(), + render.width(), + render.height()); } } @@ -556,19 +630,31 @@ namespace cmvr { if (camera_config.world_id() != config_.world_id()) { return false; } - - std::lock_guard lock(mtx_); - if (has_pip_camera_config_) { - CMVR_LOG(WARNING) << "[MujocoViewerDevice] PiP camera already configured, keep first" - << ", viewer_id=" << id_ - << ", current_camera=" << pip_camera_config_.camera_name() - << ", ignored_camera=" << camera_config.camera_name(); + if (camera_config.camera_name().empty()) { return false; } - pip_camera_config_ = camera_config; - has_pip_camera_config_ = true; - CMVR_LOG(INFO) << "[MujocoViewerDevice] set PiP camera" + std::lock_guard lock(mtx_); + if (running_) { + CMVR_LOG(WARNING) << "[MujocoViewerDevice] cannot add PiP camera while viewer is running" + << ", viewer_id=" << id_ + << ", camera=" << camera_config.camera_name(); + return false; + } + const auto duplicate = std::find_if( + pip_camera_configs_.begin(), pip_camera_configs_.end(), + [&camera_config](const config::MujocoCameraConfig& configured) { + return configured.camera_name() == camera_config.camera_name(); + }); + if (duplicate != pip_camera_configs_.end()) { + CMVR_LOG(WARNING) << "[MujocoViewerDevice] PiP camera already configured" + << ", viewer_id=" << id_ + << ", camera=" << camera_config.camera_name(); + return false; + } + + pip_camera_configs_.push_back(camera_config); + CMVR_LOG(INFO) << "[MujocoViewerDevice] add PiP camera" << ", viewer_id=" << id_ << ", camera=" << camera_config.camera_name() << ", world_id=" << camera_config.world_id(); @@ -587,6 +673,20 @@ namespace cmvr { return viewer_->getPiPCameraRGBD(rgb, depth, width, height, frame_id); } + bool MujocoViewerDevice::getPiPCameraRGBD(const std::string& camera_name, + std::vector& rgb, + std::vector& depth, + int& width, + int& height, + uint64_t& frame_id) const { + std::lock_guard lock(mtx_); + if (!viewer_) { + return false; + } + return viewer_->getPiPCameraRGBD( + camera_name, rgb, depth, width, height, frame_id); + } + bool MujocoViewerDevice::stop() { std::thread thread_to_join; { diff --git a/cmvr-es/simulate/mujoco/mujoco_viewer/src/mujoco_viewer_test.cpp b/cmvr-es/simulate/mujoco/mujoco_viewer/src/mujoco_viewer_test.cpp index 4cf3364e..6c27fe1f 100644 --- a/cmvr-es/simulate/mujoco/mujoco_viewer/src/mujoco_viewer_test.cpp +++ b/cmvr-es/simulate/mujoco/mujoco_viewer/src/mujoco_viewer_test.cpp @@ -1,11 +1,21 @@ +#include +#include #include #include #include #include #include +#include #include +#include "common/config/config_files.h" +#include "cmvr/config/device_manager_config/device_manager_config.pb.h" +#include "devices/arm/robot_arm.h" +#include "devices/camera/abstract_camera.h" +#include "devices/camera/mujoco_camera/include/mujoco_camera.h" +#include "devices/motor/manager/include/motor_manager.h" +#include "manager/device_manager/include/device_manager.h" #include "simulate/mujoco/mujoco_viewer/include/mujoco_viewer.h" #include "simulate/mujoco/mujoco_world/include/mujoco_world.h" @@ -37,6 +47,63 @@ std::string defaultModelPath() return (root / "model/xiaoyan_description/dual_arm.xml").string(); } +cmvr::device::DeviceManager& createEyeToHandDeviceManager( + const std::filesystem::path& project_root) +{ + cmvr::device::DeviceManager::destroyInstance(); + cmvr::ConfigHelper::setConfigRootFromFile( + (project_root / "cmvr-es/config/cmvr_es.pb.txt").string()); + + cmvr::config::DeviceManagerConfig config; + config.set_name("mujoco_viewer_eye_to_hand_test"); + config.set_version("test"); + + auto* world = config.add_devices(); + world->set_id("mujoco_world"); + world->set_type(cmvr::config::DeviceConfigEntry::DEVICE_TYPE_MUJOCO_WORLD); + world->set_config_file("devices/mujoco/right_arm_eye_to_hand_world.pb.txt"); + world->set_enable(true); + + auto* motors = config.add_devices(); + motors->set_id("right_arm_mujoco_motors"); + motors->set_type(cmvr::config::DeviceConfigEntry::DEVICE_TYPE_MOTOR_SYSTEM); + motors->set_config_file("devices/motor/mujoco_motors.pb.txt"); + motors->set_enable(true); + + auto* arm = config.add_devices(); + arm->set_id("mujoco_right_arm"); + arm->set_type(cmvr::config::DeviceConfigEntry::DEVICE_TYPE_ROBOT_ARM); + arm->set_config_file("devices/arm/arm_mujoco.pb.txt"); + arm->set_enable(true); + + for (const auto* camera_id : {"mujoco_hand_cam", "mujoco_external_touch_cam"}) { + auto* camera = config.add_devices(); + camera->set_id(camera_id); + camera->set_type(cmvr::config::DeviceConfigEntry::DEVICE_TYPE_CAMERA); + camera->set_config_file("devices/camera/camera.pb.txt"); + camera->set_enable(true); + } + + return cmvr::device::DeviceManager::getInstance(config); +} + +cmvr::device::Result moveArmToInitialization(cmvr::device::RobotArm& arm) +{ + const std::vector positions = { + -0.2423, + 1.2929, + 1.61, + 1.58, + -2.8792, + 0.1150, + -0.08, + }; + cmvr::device::MotionOptions options; + options.velocity = 2.8; + options.acceleration = 20.0; + return arm.moveJ(cmvr::device::JointPositionCommand{positions}, options); +} + } // namespace TEST(MujocoViewerTest, ShowsUiWithMujocoWorld) @@ -63,3 +130,67 @@ TEST(MujocoViewerTest, ShowsUiWithMujocoWorld) world->stop(); } + +TEST(MujocoViewerTest, ShowsRightArmEyeToHandCamera) +{ + const auto project_root = findProjectRoot(); + ASSERT_FALSE(project_root.empty()); + const auto model_path = project_root / "model/xiaoyan_description/right_arm_eye_to_hand.xml"; + + auto& device_manager = createEyeToHandDeviceManager(project_root); + auto arm = device_manager.getDevice("mujoco_right_arm"); + ASSERT_NE(arm, nullptr); + + auto hand_camera_base = + device_manager.getDevice("mujoco_hand_cam"); + auto external_camera_base = + device_manager.getDevice("mujoco_external_touch_cam"); + auto hand_camera = std::dynamic_pointer_cast(hand_camera_base); + auto external_camera = + std::dynamic_pointer_cast(external_camera_base); + ASSERT_NE(hand_camera, nullptr); + ASSERT_NE(external_camera, nullptr); + + auto world = cmvr::device::MotorManager::mujocoWorldFor("right_arm_mujoco_motors"); + ASSERT_NE(world, nullptr); + ASSERT_TRUE(world->isLoaded()); + ASSERT_TRUE(world->isRunning()); + + const auto move_result = moveArmToInitialization(*arm); + ASSERT_TRUE(move_result.ok()) << move_result.message; + + cmvr::MuJocoViewer viewer(world); + ASSERT_NE(viewer.model(), nullptr); + ASSERT_GE(mj_name2id(viewer.model(), mjOBJ_CAMERA, "hand_cam"), 0) + << "right_arm_eye_to_hand.xml does not contain hand_cam"; + ASSERT_GE(mj_name2id(viewer.model(), mjOBJ_CAMERA, "external_touch_cam"), 0) + << "right_arm_eye_to_hand.xml does not contain external_touch_cam"; + + viewer.setupCamera(2.5, -160.0, -25.0); + const auto& external_config = external_camera->config(); + const auto& hand_config = hand_camera->config(); + ASSERT_TRUE(external_config.viewer_pip().enable()); + ASSERT_TRUE(hand_config.viewer_pip().enable()); + viewer.enablePiPCamera(external_config.camera_name().c_str(), + external_config.viewer_pip().left(), + external_config.viewer_pip().bottom(), + external_config.viewer_pip().width(), + external_config.viewer_pip().height(), + external_config.encoder().width(), + external_config.encoder().height()); + viewer.addPiPCamera(hand_config.camera_name().c_str(), + hand_config.viewer_pip().left(), + hand_config.viewer_pip().bottom(), + hand_config.viewer_pip().width(), + hand_config.viewer_pip().height(), + hand_config.encoder().width(), + hand_config.encoder().height()); + + std::cout << "MujocoWorld loaded: " << model_path.string() << std::endl; + std::cout << "Showing hand_cam and external_touch_cam in PiP. " + "Close the MuJoCo window to exit." + << std::endl; + viewer.run(); + + world->stop(); +} diff --git a/cmvr-es/task/touch_screen_task/include/touch_screen_task.h b/cmvr-es/task/touch_screen_task/include/touch_screen_task.h index 6e284d7e..29f726c0 100644 --- a/cmvr-es/task/touch_screen_task/include/touch_screen_task.h +++ b/cmvr-es/task/touch_screen_task/include/touch_screen_task.h @@ -13,13 +13,14 @@ #include #include "cmvr/config/touch_screen_task_config/touch_screen_task_config.pb.h" -#include "algorithms/controllers/ibvs/include/ibvs_controller.h" +#include "algorithms/controllers/pbvs/include/pbvs_controller.h" +#include "algorithms/perception/apriltag/include/apriltag_perception.h" +#include "algorithms/perception/apriltag/include/tag_relative_target_3d.h" +#include "algorithms/perception/apriltag/include/tag_relative_tcp_pose.h" #include "devices/camera/abstract_camera.h" #include "devices/dexhand/abstract_dexhand.h" #include "devices/arm/robot_arm.h" #include "task/task.h" -#include "algorithms/perception/apriltag/include/apriltag_perception.h" -#include "algorithms/perception/apriltag/include/tag_relative_target_3d.h" namespace cmvr::task { @@ -27,7 +28,7 @@ class TouchScreenTask : public Task { public: enum class Phase { IDLE = 0, // 空闲,尚未开始任务。 - ALIGNING, // 视觉对准阶段:持续 IBVS 对齐目标点。 + ALIGNING, // 视觉对准阶段:持续 PBVS 对齐目标点。 ALIGN_REACHED, // 视觉对准已达到阈值,等待进入下一阶段。 TOUCHING, // 前进触控阶段:沿设定方向向屏幕推进。 DWELLING, // 已检测到接触,保持当前位置短暂停留。 @@ -40,11 +41,10 @@ public: IDLE = 0, // 空闲状态。 NOT_INITIALIZED, // 尚未调用 init() 完成初始化。 INVALID_CONFIG, // 配置非法,无法启动或应用参数。 - CONTROL_JOINT_MISMATCH, // 控制关节顺序与 IK 链不一致。 ALIGN_WAITING_PERCEPTION, // 对准阶段等待相机/AprilTag 感知结果。 ALIGN_WAITING_TRACK, // 对准阶段等待目标点跟踪恢复成功。 - ALIGN_TARGET_SETUP_FAILED,// 视觉目标设置失败,setTargetFromPointInTag 失败。 - ALIGN_COMPUTE_FAILED, // 对准阶段 IBVS 或 IK 计算失败。 + ALIGN_TARGET_SETUP_FAILED,// PBVS 目标位姿设置失败。 + ALIGN_COMPUTE_FAILED, // 对准阶段 PBVS 计算失败。 ALIGN_TIMEOUT, // 对准阶段超时仍未收敛。 ALIGNING, // 正在执行视觉对准。 ALIGN_REACHED, // 视觉对准完成。 @@ -67,6 +67,10 @@ public: bool init(const std::shared_ptr& arm, const std::shared_ptr& dexhand, const std::shared_ptr& camera); + bool init(const std::shared_ptr& arm, + const std::shared_ptr& dexhand, + const std::shared_ptr& camera, + const std::shared_ptr& external_camera); const std::string& id() const override { return id_; } @@ -92,11 +96,15 @@ public: double lastTouchPressureSum() const; int lastTouchNonzeroCount() const; int lastActiveTagId() const; - Eigen::Vector3d lastAlignErrorCamera() const; + Eigen::Vector3d lastAlignErrorScreenTag() const; const std::shared_ptr& perception() const { return perception_; } + const std::shared_ptr& externalPerception() const { + return external_perception_; + } const perception::TagRelativeTarget3D& tracker() const { return tracker_; } - const IbvsController& ibvs() const { return ibvs_; } + const perception::TagRelativeTcpPose& tcpPoseTracker() const { return tcp_pose_tracker_; } + const PbvsController& pbvs() const { return pbvs_; } private: using Clock = std::chrono::steady_clock; @@ -106,17 +114,12 @@ private: bool startFromPixelUnlocked(int u, int v); void stopUnlocked(); bool applyConfig(); - bool validateControlJointNames() const; bool stepAligning(double dt); bool stepTouching(); bool stepDwelling(); bool stepRetracting(); - bool readControlledJointPositions(std::vector& q_out) const; - bool sendJointVelocity(const std::vector& qdot) const; - bool sendZeroJointVelocity() const; - void hardStopIbvsMotion(); - bool holdCurrentControlledPosition() const; + void stopPbvsMotion(); bool buildInitJointPositions(std::vector& positions_out) const; bool moveToInitPositionBeforeStartIfEnabled(); bool moveToInitPositionIfEnabled() const; @@ -136,10 +139,13 @@ private: std::shared_ptr arm_{nullptr}; std::shared_ptr dexhand_{nullptr}; std::shared_ptr camera_{nullptr}; + std::shared_ptr external_camera_{nullptr}; std::shared_ptr perception_{nullptr}; + std::shared_ptr external_perception_{nullptr}; perception::TagRelativeTarget3D tracker_; - IbvsController ibvs_; + perception::TagRelativeTcpPose tcp_pose_tracker_; + PbvsController pbvs_; cmvr::config::TouchScreenTaskConfig config_{}; bool config_valid_{false}; @@ -149,7 +155,7 @@ private: bool initialized_{false}; bool target_locked_{false}; - bool ibvs_target_initialized_{false}; + bool pbvs_target_initialized_{false}; bool touch_command_started_{false}; bool retract_command_started_{false}; @@ -157,17 +163,24 @@ private: int target_v_{-1}; int align_stable_count_{0}; int align_debug_count_{0}; + int pbvs_debug_count_{0}; int last_active_tag_id_{-1}; double last_touch_pressure_sum_{0.0}; int last_touch_nonzero_count_{0}; - Eigen::Vector3d last_align_error_camera_{Eigen::Vector3d::Zero()}; + Eigen::Vector3d last_align_error_screen_tag_{Eigen::Vector3d::Zero()}; + Eigen::Matrix4d T_H_P_{Eigen::Matrix4d::Identity()}; + double pbvs_command_acceleration_{0.25}; bool locked_target_rotation_valid_{false}; Eigen::Matrix3d locked_target_rotation_{Eigen::Matrix3d::Identity()}; bool touch_start_position_valid_{false}; Eigen::Vector3d touch_start_position_base_{Eigen::Vector3d::Zero()}; bool retract_start_position_valid_{false}; Eigen::Vector3d retract_start_position_base_{Eigen::Vector3d::Zero()}; + bool have_last_T_B_G_{false}; + Eigen::Matrix4d last_T_B_G_{Eigen::Matrix4d::Identity()}; + double max_T_B_G_translation_delta_m_{0.0}; + double max_T_B_G_rotation_delta_rad_{0.0}; Clock::time_point phase_start_time_{}; Clock::time_point last_retract_log_time_{}; diff --git a/cmvr-es/task/touch_screen_task/src/touch_screen_task.cpp b/cmvr-es/task/touch_screen_task/src/touch_screen_task.cpp index fc493498..8351a848 100644 --- a/cmvr-es/task/touch_screen_task/src/touch_screen_task.cpp +++ b/cmvr-es/task/touch_screen_task/src/touch_screen_task.cpp @@ -7,12 +7,14 @@ #include #include +#include + #include "common/base/logging/logger.h" +#include "common/math/cartesian_motion_math.h" #include "common/math/proto_geometry.h" -#include "algorithms/kinematics/ik_solver/pinocchio/include/pinocchio_ik_base.h" +#include "common/math/transform_math.h" #include "cmvr/config/touch_screen_algorithm_config.pb.h" #include "manager/device_manager/include/device_manager.h" -#include namespace cmvr::task { @@ -54,15 +56,8 @@ using TouchScreenTaskConfig = cmvr::config::TouchScreenTaskConfig; std::array toArray6(const Eigen::Matrix& value) { return {{value[0], value[1], value[2], value[3], value[4], value[5]}}; } - - - -std::vector controlJointNames(const TouchScreenTaskConfig& config) { - const auto& names = config.alignment().ibvs().control_joint_names(); - return {names.begin(), names.end()}; -} - bool buildInitJointPositionsFromConfig(const TouchScreenTaskConfig& config, + const std::vector& joint_names, std::vector& positions_out) { std::unordered_map q_map; q_map.reserve(static_cast(config.initialization().joint_positions_size())); @@ -71,13 +66,14 @@ bool buildInitJointPositionsFromConfig(const TouchScreenTaskConfig& config, !std::isfinite(joint.rad())) { return false; } - q_map[joint.joint_name()] = joint.rad(); + if (!q_map.emplace(joint.joint_name(), joint.rad()).second) { + return false; + } } positions_out.clear(); - positions_out.reserve(static_cast( - config.alignment().ibvs().control_joint_names_size())); - for (const auto& name : config.alignment().ibvs().control_joint_names()) { + positions_out.reserve(joint_names.size()); + for (const auto& name : joint_names) { const auto it = q_map.find(name); if (it == q_map.end()) { return false; @@ -87,6 +83,30 @@ bool buildInitJointPositionsFromConfig(const TouchScreenTaskConfig& config, return !positions_out.empty(); } +bool hasMat4(const cmvr::common::Mat4& value) { + return value.has_m00() && value.has_m01() && value.has_m02() && value.has_m03() && + value.has_m10() && value.has_m11() && value.has_m12() && value.has_m13() && + value.has_m20() && value.has_m21() && value.has_m22() && value.has_m23() && + value.has_m30() && value.has_m31() && value.has_m32() && value.has_m33(); +} + +Eigen::Matrix4d toEigenMat4(const cmvr::common::Mat4& value) { + Eigen::Matrix4d transform; + transform << value.m00(), value.m01(), value.m02(), value.m03(), + value.m10(), value.m11(), value.m12(), value.m13(), + value.m20(), value.m21(), value.m22(), value.m23(), + value.m30(), value.m31(), value.m32(), value.m33(); + return transform; +} + +bool isHomogeneousTransform(const Eigen::Matrix4d& transform) { + return transform.allFinite() && + std::abs(transform(3, 0)) <= 1e-6 && + std::abs(transform(3, 1)) <= 1e-6 && + std::abs(transform(3, 2)) <= 1e-6 && + std::abs(transform(3, 3) - 1.0) <= 1e-6; +} + bool isTouchTriggered(const TouchScreenTaskConfig& config, const double resultant_force_value) { return resultant_force_value >= config.touch().tactile().force_threshold(); @@ -103,36 +123,15 @@ double tactileForceValue(const device::AbstractDexHand::TactilePoint& point, return static_cast(point.fz); } -Eigen::Matrix3d rotationFromTargetRotvec(double rx, double ry, double rz) { - vpRotationMatrix R_visp; - R_visp.buildFrom(rx, ry, rz); - - Eigen::Matrix3d R = Eigen::Matrix3d::Identity(); - for (int r = 0; r < 3; ++r) { - for (int c = 0; c < 3; ++c) { - R(r, c) = R_visp[r][c]; - } +Eigen::Matrix3d rotationFromTargetRotvec(const double rx, + const double ry, + const double rz) { + const Eigen::Vector3d rotation_vector(rx, ry, rz); + const double angle = rotation_vector.norm(); + if (!std::isfinite(angle) || angle <= 1e-12) { + return Eigen::Matrix3d::Identity(); } - return R; -} - -double rotationErrorRad(const Eigen::Matrix3d& R_current, - const Eigen::Matrix3d& R_target) { - if (!R_current.allFinite() || !R_target.allFinite()) { - return std::numeric_limits::infinity(); - } - - const Eigen::Matrix3d R_err = R_current * R_target.transpose(); - const double cos_angle = std::clamp(0.5 * (R_err.trace() - 1.0), -1.0, 1.0); - return std::acos(cos_angle); -} - -Eigen::Vector3d rotvecFromRotationMatrix(const Eigen::Matrix3d& R) { - const Eigen::AngleAxisd aa(R); - if (!std::isfinite(aa.angle()) || !aa.axis().allFinite() || std::abs(aa.angle()) <= 1e-12) { - return Eigen::Vector3d::Zero(); - } - return aa.axis() * aa.angle(); + return Eigen::AngleAxisd(angle, rotation_vector / angle).toRotationMatrix(); } bool extractProjectedYawAboutTargetNormal(const Eigen::Matrix3d& R_target, @@ -285,6 +284,7 @@ bool TouchScreenTask::init() { const auto& devices = config_.devices(); if (!devices.has_arm_id() || devices.arm_id().empty() || !devices.has_camera_id() || devices.camera_id().empty() || + !devices.has_external_camera_id() || devices.external_camera_id().empty() || !devices.has_dexhand_id() || devices.dexhand_id().empty()) { last_status_ = Status::INVALID_CONFIG; return false; @@ -294,23 +294,44 @@ bool TouchScreenTask::init() { auto arm = dm.getDevice(devices.arm_id()); auto dexhand = dm.getDevice(devices.dexhand_id()); auto camera = dm.getDevice(devices.camera_id()); + auto external_camera = + dm.getDevice(devices.external_camera_id()); if (!camera || !camera->start()) { CMVR_LOG(ERROR) << "[TouchScreenTask] Failed to start camera: " << devices.camera_id(); last_status_ = Status::NOT_INITIALIZED; return false; } - return init(arm, dexhand, camera); + if (!external_camera || !external_camera->start()) { + CMVR_LOG(ERROR) << "[TouchScreenTask] Failed to start external camera: " + << devices.external_camera_id(); + last_status_ = Status::NOT_INITIALIZED; + return false; + } + return init(arm, dexhand, camera, external_camera); } bool TouchScreenTask::init(const std::shared_ptr& arm, const std::shared_ptr& dexhand, const std::shared_ptr& camera) { + std::shared_ptr external_camera; + if (config_.has_devices() && config_.devices().has_external_camera_id()) { + external_camera = device::DeviceManager::getInstance().getDevice( + config_.devices().external_camera_id()); + } + return init(arm, dexhand, camera, external_camera); +} + +bool TouchScreenTask::init(const std::shared_ptr& arm, + const std::shared_ptr& dexhand, + const std::shared_ptr& camera, + const std::shared_ptr& external_camera) { std::lock_guard lock(mutex_); arm_ = arm; dexhand_ = dexhand; camera_ = camera; + external_camera_ = external_camera; - if (!config_valid_ || !arm_ || !camera_) { + if (!config_valid_ || !arm_ || !camera_ || !external_camera_) { initialized_ = false; last_status_ = Status::INVALID_CONFIG; return false; @@ -323,58 +344,47 @@ bool TouchScreenTask::init(const std::shared_ptr& arm, return false; } - if (config_.alignment().ibvs().camera_link().empty() || - config_.alignment().ibvs().control_joint_names().empty()) { - initialized_ = false; - last_status_ = Status::INVALID_CONFIG; - return false; - } - perception_ = std::make_shared(camera_); perception_->setTagSize(config_.perception().apriltag().tag_size_m()); + external_perception_ = + std::make_shared(external_camera_); + external_perception_->setTagSize( + config_.perception().external_apriltag().tag_size_m()); tracker_.setPerception(perception_); tracker_.setTargetPointMethod(toTargetPointMethod( config_.perception().apriltag().target_point_method())); - auto pinocchio_solver = - std::dynamic_pointer_cast(arm_->kinematicsSolver()); - if (!pinocchio_solver || - !ibvs_.init(pinocchio_solver, - config_.alignment().ibvs().camera_link())) { - initialized_ = false; - last_status_ = Status::INVALID_CONFIG; - return false; - } - - ibvs_.setPerception(perception_); - if (!validateControlJointNames()) { - initialized_ = false; - last_status_ = Status::CONTROL_JOINT_MISMATCH; - return false; - } + tcp_pose_tracker_.setPerception(external_perception_); + tcp_pose_tracker_.setHandTagId( + config_.perception().external_apriltag().hand_tag_id()); initialized_ = applyConfig(); if (initialized_) { tracker_.clear(); tracker_.resetActiveTagTracking(); - ibvs_.reset(); + tcp_pose_tracker_.clear(); + pbvs_.reset(); phase_ = Phase::IDLE; phase_after_retract_ = Phase::DONE; final_status_after_retract_ = Status::DONE; target_locked_ = false; - ibvs_target_initialized_ = false; + pbvs_target_initialized_ = false; touch_command_started_ = false; retract_command_started_ = false; align_stable_count_ = 0; last_active_tag_id_ = -1; last_touch_pressure_sum_ = 0.0; last_touch_nonzero_count_ = 0; - last_align_error_camera_.setZero(); + last_align_error_screen_tag_.setZero(); touch_start_position_valid_ = false; touch_start_position_base_.setZero(); retract_start_position_valid_ = false; retract_start_position_base_.setZero(); + have_last_T_B_G_ = false; + last_T_B_G_.setIdentity(); + max_T_B_G_translation_delta_m_ = 0.0; + max_T_B_G_rotation_delta_rad_ = 0.0; last_status_ = Status::IDLE; } else { last_status_ = Status::INVALID_CONFIG; @@ -413,26 +423,32 @@ bool TouchScreenTask::startFromPixelUnlocked(int u, int v) { tracker_.clear(); tracker_.resetActiveTagTracking(); - ibvs_.reset(); + tcp_pose_tracker_.clear(); + pbvs_.reset(); target_u_ = u; target_v_ = v; target_locked_ = false; - ibvs_target_initialized_ = false; + pbvs_target_initialized_ = false; touch_command_started_ = false; retract_command_started_ = false; align_stable_count_ = 0; align_debug_count_ = 0; + pbvs_debug_count_ = 0; last_touch_pressure_sum_ = 0.0; last_touch_nonzero_count_ = 0; last_active_tag_id_ = -1; - last_align_error_camera_.setZero(); + last_align_error_screen_tag_.setZero(); locked_target_rotation_valid_ = false; locked_target_rotation_.setIdentity(); touch_start_position_valid_ = false; touch_start_position_base_.setZero(); retract_start_position_valid_ = false; retract_start_position_base_.setZero(); + have_last_T_B_G_ = false; + last_T_B_G_.setIdentity(); + max_T_B_G_translation_delta_m_ = 0.0; + max_T_B_G_rotation_delta_rad_ = 0.0; phase_ = Phase::ALIGNING; phase_after_retract_ = Phase::DONE; final_status_after_retract_ = Status::DONE; @@ -512,28 +528,30 @@ void TouchScreenTask::stopUnlocked() { } } - sendZeroJointVelocity(); - ibvs_.resetTwistCommandState(); - holdCurrentControlledPosition(); + pbvs_.resetTwistCommandState(); phase_ = Phase::IDLE; phase_after_retract_ = Phase::DONE; final_status_after_retract_ = Status::DONE; target_locked_ = false; - ibvs_target_initialized_ = false; + pbvs_target_initialized_ = false; touch_command_started_ = false; retract_command_started_ = false; align_stable_count_ = 0; last_active_tag_id_ = -1; last_touch_pressure_sum_ = 0.0; last_touch_nonzero_count_ = 0; - last_align_error_camera_.setZero(); + last_align_error_screen_tag_.setZero(); locked_target_rotation_valid_ = false; locked_target_rotation_.setIdentity(); touch_start_position_valid_ = false; touch_start_position_base_.setZero(); retract_start_position_valid_ = false; retract_start_position_base_.setZero(); + have_last_T_B_G_ = false; + last_T_B_G_.setIdentity(); + max_T_B_G_translation_delta_m_ = 0.0; + max_T_B_G_rotation_delta_rad_ = 0.0; last_status_ = Status::STOPPED; } @@ -613,9 +631,9 @@ int TouchScreenTask::lastActiveTagId() const { return last_active_tag_id_; } -Eigen::Vector3d TouchScreenTask::lastAlignErrorCamera() const { +Eigen::Vector3d TouchScreenTask::lastAlignErrorScreenTag() const { std::lock_guard lock(mutex_); - return last_align_error_camera_; + return last_align_error_screen_tag_; } std::string TouchScreenTask::stateString() const { @@ -646,7 +664,6 @@ const char* TouchScreenTask::statusToString(const Status status) { case Status::IDLE: return "IDLE"; case Status::NOT_INITIALIZED: return "NOT_INITIALIZED"; case Status::INVALID_CONFIG: return "INVALID_CONFIG"; - case Status::CONTROL_JOINT_MISMATCH: return "CONTROL_JOINT_MISMATCH"; case Status::ALIGN_WAITING_PERCEPTION: return "ALIGN_WAITING_PERCEPTION"; case Status::ALIGN_WAITING_TRACK: return "ALIGN_WAITING_TRACK"; case Status::ALIGN_TARGET_SETUP_FAILED: return "ALIGN_TARGET_SETUP_FAILED"; @@ -674,6 +691,8 @@ bool TouchScreenTask::validateConfig(const cmvr::config::TouchScreenTaskConfig& !config.devices().has_arm_id() || config.devices().arm_id().empty() || !config.devices().has_dexhand_id() || config.devices().dexhand_id().empty() || !config.devices().has_camera_id() || config.devices().camera_id().empty() || + !config.devices().has_external_camera_id() || + config.devices().external_camera_id().empty() || !config.has_initialization() || !config.initialization().has_before_start() || !config.initialization().has_after_finish() || @@ -681,8 +700,9 @@ bool TouchScreenTask::validateConfig(const cmvr::config::TouchScreenTaskConfig& !config.initialization().has_acceleration() || !config.has_perception() || !config.perception().has_apriltag() || + !config.perception().has_external_apriltag() || !config.has_alignment() || - !config.alignment().has_ibvs() || + !config.alignment().has_pbvs() || !config.alignment().has_target() || !config.alignment().has_error_threshold() || !config.alignment().has_stable_frames() || @@ -699,8 +719,9 @@ bool TouchScreenTask::validateConfig(const cmvr::config::TouchScreenTaskConfig& } const auto& apriltag = config.perception().apriltag(); + const auto& external_apriltag = config.perception().external_apriltag(); const auto& alignment = config.alignment(); - const auto& ibvs = alignment.ibvs(); + const auto& pbvs = alignment.pbvs(); const auto& target = alignment.target(); const auto& touch = config.touch(); const auto& tactile = touch.tactile(); @@ -709,25 +730,23 @@ bool TouchScreenTask::validateConfig(const cmvr::config::TouchScreenTaskConfig& if (!apriltag.has_tag_size_m() || !apriltag.has_depth_policy() || !apriltag.has_target_point_method() || - !target.has_position_in_camera() || - !hasVec3(target.position_in_camera()) || + !external_apriltag.has_tag_size_m() || + !external_apriltag.has_hand_tag_id() || + !external_apriltag.has_t_h_p() || + !hasMat4(external_apriltag.t_h_p()) || !target.has_rotation_vector() || !hasVec3(target.rotation_vector()) || !target.has_mode() || !hasVec6(alignment.error_threshold()) || - !ibvs.has_camera_link() || - !ibvs.has_lambda() || - !ibvs.has_mu() || - !ibvs.has_qdot_max() || - !ibvs.has_vmax6() || - !hasVec6(ibvs.vmax6()) || - !ibvs.has_amax6() || - !hasVec6(ibvs.amax6()) || - !ibvs.has_twist_filter_alpha() || - !ibvs.has_r_camera_to_visp() || - !hasMat3(ibvs.r_camera_to_visp()) || - !ibvs.has_r_camera_to_urdf() || - !hasMat3(ibvs.r_camera_to_urdf()) || + !pbvs.has_position_gain() || + !hasVec3(pbvs.position_gain()) || + !pbvs.has_rotation_gain() || + !hasVec3(pbvs.rotation_gain()) || + !pbvs.has_vmax6() || + !hasVec6(pbvs.vmax6()) || + !pbvs.has_amax6() || + !hasVec6(pbvs.amax6()) || + !pbvs.has_twist_filter_alpha() || !tactile.has_finger() || !tactile.has_region() || !tactile.has_criterion() || @@ -736,37 +755,49 @@ bool TouchScreenTask::validateConfig(const cmvr::config::TouchScreenTaskConfig& return false; } - if (ibvs.camera_link().empty() || - ibvs.control_joint_names().empty()) { - return false; - } - - const Eigen::Vector3d target_position = - cmvr::common::math::toEigenVec3(alignment.target().position_in_camera()); const Eigen::Vector3d target_rotation = cmvr::common::math::toEigenVec3(alignment.target().rotation_vector()); const Eigen::Matrix error_threshold = cmvr::common::math::toEigenVec6(alignment.error_threshold()); - const Eigen::Matrix3d camera_to_visp = - cmvr::common::math::toEigenMat3(ibvs.r_camera_to_visp()); - const Eigen::Matrix3d camera_to_urdf = - cmvr::common::math::toEigenMat3(ibvs.r_camera_to_urdf()); + const Eigen::Vector3d position_gain = + cmvr::common::math::toEigenVec3(pbvs.position_gain()); + const Eigen::Vector3d rotation_gain = + cmvr::common::math::toEigenVec3(pbvs.rotation_gain()); + const Eigen::Matrix vmax6 = + cmvr::common::math::toEigenVec6(pbvs.vmax6()); + const Eigen::Matrix amax6 = + cmvr::common::math::toEigenVec6(pbvs.amax6()); + const Eigen::Matrix4d T_H_P = toEigenMat4(external_apriltag.t_h_p()); if (!std::isfinite(apriltag.tag_size_m()) || apriltag.tag_size_m() <= 0.0 || - !target_position.allFinite() || + !std::isfinite(external_apriltag.tag_size_m()) || + external_apriltag.tag_size_m() <= 0.0 || + external_apriltag.hand_tag_id() < 0 || + !isHomogeneousTransform(T_H_P) || !target_rotation.allFinite() || + !position_gain.allFinite() || + !rotation_gain.allFinite() || + !vmax6.allFinite() || + !amax6.allFinite() || + !std::isfinite(pbvs.twist_filter_alpha()) || + pbvs.twist_filter_alpha() < 0.0 || pbvs.twist_filter_alpha() > 1.0 || alignment.stable_frames() <= 0 || !std::isfinite(alignment.timeout_s()) || alignment.timeout_s() <= 0.0 || !std::isfinite(tactile.force_threshold()) || tactile.force_threshold() < 0.0 || !std::isfinite(touch.dwell_time_s()) || touch.dwell_time_s() < 0.0 || - !camera_to_visp.allFinite() || !camera_to_urdf.allFinite() || !cmvr::common::math::toEigenVec6(retract.twist_tool()).allFinite() || !std::isfinite(retract.acceleration()) || retract.acceleration() <= 0.0 || !std::isfinite(retract.duration_s()) || retract.duration_s() < 0.0) { return false; } - for (int i = 0; i < error_threshold.size(); ++i) { - if (!std::isfinite(error_threshold[i]) || error_threshold[i] < 0.0) { + for (int i = 0; i < 3; ++i) { + if (position_gain[i] < 0.0 || rotation_gain[i] < 0.0) { + return false; + } + } + for (int i = 0; i < 6; ++i) { + if (!std::isfinite(error_threshold[i]) || error_threshold[i] < 0.0 || + vmax6[i] <= 0.0 || amax6[i] < 0.0) { return false; } } @@ -779,8 +810,16 @@ bool TouchScreenTask::validateConfig(const cmvr::config::TouchScreenTaskConfig& } if (config.initialization().before_start() || config.initialization().after_finish()) { + std::vector init_joint_names; + init_joint_names.reserve(static_cast( + config.initialization().joint_positions_size())); + for (const auto& joint : config.initialization().joint_positions()) { + init_joint_names.push_back(joint.joint_name()); + } std::vector init_positions; - if (!buildInitJointPositionsFromConfig(config, init_positions)) { + if (!buildInitJointPositionsFromConfig(config, init_joint_names, init_positions) || + static_cast(init_positions.size()) != + config.initialization().joint_positions_size()) { return false; } } @@ -826,7 +865,7 @@ bool TouchScreenTask::validateConfig(const cmvr::config::TouchScreenTaskConfig& !std::isfinite(move_l.velocity()) || move_l.velocity() <= 0.0 || !std::isfinite(move_l.acceleration()) || move_l.acceleration() <= 0.0 || !std::isfinite(move_l.jerk()) || move_l.jerk() <= 0.0 || - move_l.joint_velocity_limits_size() != ibvs.control_joint_names_size()) { + move_l.joint_velocity_limits().empty()) { return false; } for (const double qd_max_i : move_l.joint_velocity_limits()) { @@ -843,7 +882,7 @@ bool TouchScreenTask::validateConfig(const cmvr::config::TouchScreenTaskConfig& } bool TouchScreenTask::applyConfig() { - if (!perception_) { + if (!perception_ || !external_perception_) { return false; } if (!validateConfig(config_)) { @@ -866,84 +905,51 @@ bool TouchScreenTask::applyConfig() { } const auto& apriltag = config_.perception().apriltag(); - const auto& ibvs = config_.alignment().ibvs(); - const auto target_position_in_camera = - cmvr::common::math::toEigenVec3(config_.alignment().target().position_in_camera()); - const Eigen::Matrix3d r_camera_to_visp = - cmvr::common::math::toEigenMat3(ibvs.r_camera_to_visp()); - const Eigen::Matrix3d r_camera_to_urdf = - cmvr::common::math::toEigenMat3(ibvs.r_camera_to_urdf()); + const auto& external_apriltag = config_.perception().external_apriltag(); + const auto& pbvs = config_.alignment().pbvs(); perception_->setTagSize(apriltag.tag_size_m()); + external_perception_->setTagSize(external_apriltag.tag_size_m()); tracker_.setTargetPointMethod(toTargetPointMethod(apriltag.target_point_method())); - ibvs_.setLambda(ibvs.lambda()); - ibvs_.setMu(ibvs.mu()); - ibvs_.setQdotMax(ibvs.qdot_max()); - ibvs_.setVelocityLimit6(toArray6(cmvr::common::math::toEigenVec6(ibvs.vmax6()))); - ibvs_.setAccelerationLimit6(toArray6(cmvr::common::math::toEigenVec6(ibvs.amax6()))); - ibvs_.setTwistFilterAlpha(ibvs.twist_filter_alpha()); - ibvs_.setAlignCameraToVisp(r_camera_to_visp); - ibvs_.setAlignCameraToUrdf(r_camera_to_urdf); + tcp_pose_tracker_.setPerception(external_perception_); + tcp_pose_tracker_.setHandTagId(external_apriltag.hand_tag_id()); + T_H_P_ = toEigenMat4(external_apriltag.t_h_p()); + + pbvs_.setPositionGain(cmvr::common::math::toEigenVec3(pbvs.position_gain())); + pbvs_.setRotationGain(cmvr::common::math::toEigenVec3(pbvs.rotation_gain())); + pbvs_.setVelocityLimit6(toArray6(cmvr::common::math::toEigenVec6(pbvs.vmax6()))); + pbvs_.setAccelerationLimit6(toArray6(cmvr::common::math::toEigenVec6(pbvs.amax6()))); + pbvs_.setTolerance6(toArray6( + cmvr::common::math::toEigenVec6(config_.alignment().error_threshold()))); + pbvs_.setTwistFilterAlpha(pbvs.twist_filter_alpha()); + + std::array enabled{{true, true, true, true, true, true}}; + if (config_.alignment().target().mode() == + cmvr::config::TOUCH_SCREEN_ALIGN_MODE_RX_RY_AND_POSITION) { + enabled[5] = false; + } else if (config_.alignment().target().mode() == + cmvr::config::TOUCH_SCREEN_ALIGN_MODE_POSITION_ONLY) { + enabled[3] = false; + enabled[4] = false; + enabled[5] = false; + } + pbvs_.setAxisEnabled(enabled); + + const auto amax = cmvr::common::math::toEigenVec6(pbvs.amax6()); + pbvs_command_acceleration_ = std::max(0.01, amax.head<3>().maxCoeff()); CMVR_LOG(DEBUG) << "[TouchScreenTask] Apply config id=" << config_.id() << ", arm_id=" << config_.devices().arm_id() - << ", camera_id=" << config_.devices().camera_id() - << ", tag_size_m=" << apriltag.tag_size_m() - << ", target_position_in_camera=[" << target_position_in_camera.x() - << ", " << target_position_in_camera.y() - << ", " << target_position_in_camera.z() << "]" - << ", r_camera_to_visp=[" << r_camera_to_visp(0, 0) - << ", " << r_camera_to_visp(0, 1) - << ", " << r_camera_to_visp(0, 2) - << "; " << r_camera_to_visp(1, 0) - << ", " << r_camera_to_visp(1, 1) - << ", " << r_camera_to_visp(1, 2) - << "; " << r_camera_to_visp(2, 0) - << ", " << r_camera_to_visp(2, 1) - << ", " << r_camera_to_visp(2, 2) << "]" - << ", r_camera_to_urdf=[" << r_camera_to_urdf(0, 0) - << ", " << r_camera_to_urdf(0, 1) - << ", " << r_camera_to_urdf(0, 2) - << "; " << r_camera_to_urdf(1, 0) - << ", " << r_camera_to_urdf(1, 1) - << ", " << r_camera_to_urdf(1, 2) - << "; " << r_camera_to_urdf(2, 0) - << ", " << r_camera_to_urdf(2, 1) - << ", " << r_camera_to_urdf(2, 2) << "]"; + << ", hand_camera_id=" << config_.devices().camera_id() + << ", external_camera_id=" << config_.devices().external_camera_id() + << ", hand_camera_tag_size_m=" << apriltag.tag_size_m() + << ", external_camera_tag_size_m=" << external_apriltag.tag_size_m() + << ", hand_tag_id=" << external_apriltag.hand_tag_id(); return true; } -bool TouchScreenTask::validateControlJointNames() const { - std::vector solver_joint_names; - if (!ibvs_.getChainJointNames(solver_joint_names)) { - return false; - } - const std::vector task_joint_names = controlJointNames(config_); - if (solver_joint_names == task_joint_names) { - return true; - } - - std::ostringstream mismatch; - mismatch << "[TouchScreenTask] control_joint_names mismatch with IbvsController IK chain" - << ", task joints=["; - for (const auto& name : task_joint_names) { - mismatch << name << ' '; - } - mismatch << "], solver joints=["; - for (const auto& name : solver_joint_names) { - mismatch << name << ' '; - } - mismatch << ']'; - CMVR_LOG(ERROR) << mismatch.str(); - return false; -} - bool TouchScreenTask::stepAligning(const double dt) { const auto& alignment = config_.alignment(); - const auto target_position_in_camera = - cmvr::common::math::toEigenVec3(alignment.target().position_in_camera()); const auto target_rotation_vector = cmvr::common::math::toEigenVec3(alignment.target().rotation_vector()); - const auto error_threshold = - cmvr::common::math::toEigenVec6(alignment.error_threshold()); const auto now = Clock::now(); const double elapsed = std::chrono::duration(now - phase_start_time_).count(); @@ -951,17 +957,42 @@ bool TouchScreenTask::stepAligning(const double dt) { enterFailed(Status::ALIGN_TIMEOUT); return false; } - const double ibvs_dt = std::clamp(dt, 0.005, 0.05); + const double pbvs_dt = std::clamp(dt, 0.0001, 0.05); - perception::AprilTagPerception::Options perception_options; - perception_options.depth_policy = - toDepthPolicy(config_.perception().apriltag().depth_policy()); - perception_options.detect_tags = true; - perception_options.fetch_encoded = false; - if (!perception_->update(perception_options)) { - hardStopIbvsMotion(); - last_status_ = Status::ALIGN_WAITING_PERCEPTION; - return true; + // The hand camera is only needed to establish the target anchor once. + // After the first successful PBVS target initialization, tracker_ keeps + // that anchor fixed and the external camera supplies the live TCP pose. + const bool hand_camera_needed = !target_locked_ || !pbvs_target_initialized_; + bool hand_perception_ok = true; + if (hand_camera_needed) { + perception::AprilTagPerception::Options hand_options; + hand_options.depth_policy = + toDepthPolicy(config_.perception().apriltag().depth_policy()); + hand_options.detect_tags = true; + hand_options.fetch_encoded = false; + hand_perception_ok = perception_->update(hand_options); + if (!hand_perception_ok) { + if (align_debug_count_ == 0 && !perception_->color().empty()) { + cv::imwrite("/tmp/cmvr_hand_camera_perception.png", perception_->color()); + } + if ((align_debug_count_++ % 200) == 0) { + const auto& image = perception_->color(); + const auto& intrinsics = perception_->intrinsics(); + CMVR_LOG(INFO) << "[TouchScreenTask][ALIGN_PERCEPTION_DEBUG]" + << " status=" + << perception::AprilTagPerception::statusToString( + perception_->lastStatus()) + << ", image=" << image.cols << "x" << image.rows + << ", channels=" << image.channels() + << ", fx=" << intrinsics.fx + << ", fy=" << intrinsics.fy + << ", cx=" << intrinsics.cx + << ", cy=" << intrinsics.cy; + } + stopPbvsMotion(); + last_status_ = Status::ALIGN_WAITING_PERCEPTION; + return true; + } } bool tracking_ok = false; @@ -969,15 +1000,28 @@ bool TouchScreenTask::stepAligning(const double dt) { tracking_ok = tracker_.startTrackingFromPixel(target_u_, target_v_); if (tracking_ok) { target_locked_ = true; - ibvs_target_initialized_ = false; + pbvs_target_initialized_ = false; align_stable_count_ = 0; } - } else { + } else if (hand_camera_needed && hand_perception_ok) { tracking_ok = tracker_.track(); } + // The touch pixel is converted into a fixed anchor in the selected tag + // frame on the first successful observation. Once locked, the external + // camera provides the live TCP pose, so the hand camera is intentionally + // skipped and cannot interrupt the PBVS motion or invalidate that target. + if (!tracking_ok && target_locked_ && pbvs_target_initialized_) { + const int locked_tag_id = tracker_.activeTagId(); + tracking_ok = locked_tag_id >= 0 && tracker_.hasAnchorForTag(locked_tag_id); + if (tracking_ok && (pbvs_debug_count_ % 20) == 0) { + CMVR_LOG(DEBUG) << "[TouchScreenTask][ALIGNING] target anchor locked; " + << "skip hand camera sampling, tag_id=" << locked_tag_id; + } + } + if (!tracking_ok) { - hardStopIbvsMotion(); + stopPbvsMotion(); align_stable_count_ = 0; last_status_ = Status::ALIGN_WAITING_TRACK; return true; @@ -986,32 +1030,65 @@ bool TouchScreenTask::stepAligning(const double dt) { const int tag_id = tracker_.activeTagId(); last_active_tag_id_ = tag_id; if (tag_id < 0) { - hardStopIbvsMotion(); + stopPbvsMotion(); align_stable_count_ = 0; last_status_ = Status::ALIGN_WAITING_TRACK; return true; } - Eigen::Vector3d p_t_target = Eigen::Vector3d::Zero(); - if (!tracker_.getAnchorInTag(tag_id, p_t_target)) { - hardStopIbvsMotion(); + Eigen::Vector3d p_target_G = Eigen::Vector3d::Zero(); + if (!tracker_.getAnchorInTag(tag_id, p_target_G)) { + stopPbvsMotion(); align_stable_count_ = 0; last_status_ = Status::ALIGN_WAITING_TRACK; return true; } - const auto* current_tag = perception_->findTag(tag_id); - if (!current_tag) { - hardStopIbvsMotion(); + perception::AprilTagPerception::Options external_options; + external_options.depth_policy = perception::AprilTagPerception::DepthPolicy::NONE; + external_options.detect_tags = true; + external_options.fetch_encoded = false; + if (!external_perception_->update(external_options)) { + stopPbvsMotion(); + align_stable_count_ = 0; + last_status_ = Status::ALIGN_WAITING_PERCEPTION; + return true; + } + + tcp_pose_tracker_.setScreenTagId(tag_id); + if (!tcp_pose_tracker_.update(T_H_P_)) { + if (align_debug_count_ == 0 && !external_perception_->color().empty()) { + cv::imwrite("/tmp/cmvr_external_camera_perception.png", + external_perception_->color()); + } + if ((align_debug_count_++ % 200) == 0) { + std::ostringstream detected_ids; + for (const auto& detected_tag : external_perception_->tags()) { + if (detected_ids.tellp() > 0) { + detected_ids << ','; + } + detected_ids << detected_tag.id; + } + CMVR_LOG(INFO) << "[TouchScreenTask][EXTERNAL_PERCEPTION_DEBUG]" + << " status=" + << perception::AprilTagPerception::statusToString( + external_perception_->lastStatus()) + << ", detected_ids=[" << detected_ids.str() << ']' + << ", tcp_status=" + << perception::TagRelativeTcpPose::statusToString( + tcp_pose_tracker_.lastStatus()); + } + stopPbvsMotion(); align_stable_count_ = 0; last_status_ = Status::ALIGN_WAITING_TRACK; return true; } + const Eigen::Matrix4d& T_G_P = tcp_pose_tracker_.T_G_P(); Eigen::Matrix3d R_target = rotationFromTargetRotvec(target_rotation_vector.x(), target_rotation_vector.y(), target_rotation_vector.z()); - const Eigen::Matrix3d R_current = current_tag->T_c_t.block<3, 3>(0, 0); + const Eigen::Matrix3d R_current = T_G_P.block<3, 3>(0, 0); switch (alignment.target().mode()) { case cmvr::config::TOUCH_SCREEN_ALIGN_MODE_POSE_AND_POSITION: break; @@ -1030,128 +1107,146 @@ bool TouchScreenTask::stepAligning(const double dt) { R_target = locked_target_rotation_; break; case cmvr::config::TOUCH_SCREEN_ALIGN_MODE_POSITION_ONLY: - // True position-only mode: keep the desired orientation equal to the - // current tag orientation every frame so IBVS does not actively try to - // correct rotational error. R_target = R_current; break; } - const Eigen::Vector3d target_rotvec = rotvecFromRotationMatrix(R_target); - ibvs_.setTrackedTagId(tag_id); - const bool refresh_target = !ibvs_target_initialized_ || tracker_.lastSwitched() || - alignment.target().mode() == - cmvr::config::TOUCH_SCREEN_ALIGN_MODE_POSITION_ONLY; + const bool refresh_target = !pbvs_target_initialized_ || tracker_.lastSwitched(); if (refresh_target) { - if (!ibvs_.setTargetFromPointInTag(p_t_target, - target_position_in_camera, - target_rotvec.x(), - target_rotvec.y(), - target_rotvec.z())) { + if (tracker_.lastSwitched()) { + pbvs_.resetTwistCommandState(); + } + if (!pbvs_.setTargetPose(p_target_G, R_target)) { enterFailed(Status::ALIGN_TARGET_SETUP_FAILED); return false; } - ibvs_target_initialized_ = true; + pbvs_target_initialized_ = true; } - std::vector q_now; - if (!readControlledJointPositions(q_now)) { + PbvsController::Output output; + if (!pbvs_.compute(T_G_P, pbvs_dt, output)) { + enterFailed(Status::ALIGN_COMPUTE_FAILED); + return false; + } + + Eigen::Matrix4d T_B_P = Eigen::Matrix4d::Identity(); + try { + T_B_P = cmvr::common::math::poseToMatrix(arm_->fk(true)); + } catch (...) { + enterFailed(Status::ROBOT_STATE_FAILED); + return false; + } + if (!isHomogeneousTransform(T_B_P)) { enterFailed(Status::ROBOT_STATE_FAILED); return false; } - std::vector qdot_cmd; - if (!ibvs_.computeQdot(q_now, ibvs_dt, qdot_cmd)) { - switch (ibvs_.lastComputeStatus()) { - case IbvsController::ComputeStatus::NO_NEW_FRAME: - case IbvsController::ComputeStatus::NO_TAG: - case IbvsController::ComputeStatus::TAG_MISMATCH: - case IbvsController::ComputeStatus::NO_DEPTH: - hardStopIbvsMotion(); - align_stable_count_ = 0; - last_status_ = Status::ALIGN_WAITING_TRACK; - return true; - case IbvsController::ComputeStatus::OK: - case IbvsController::ComputeStatus::NOT_READY: - case IbvsController::ComputeStatus::BAD_IMAGE: - case IbvsController::ComputeStatus::INVALID_INPUT: - case IbvsController::ComputeStatus::IK_FAILED: - default: - enterFailed(Status::ALIGN_COMPUTE_FAILED); - return false; - } + const Eigen::Matrix4d T_B_G = T_B_P * T_G_P.inverse(); + if (!isHomogeneousTransform(T_B_G)) { + enterFailed(Status::ALIGN_COMPUTE_FAILED); + return false; } - - if ((align_debug_count_++ % 20) == 0) { - Eigen::Matrix achieved_twist_base = - Eigen::Matrix::Zero(); - bool achieved_ok = false; - auto pinocchio_solver = - arm_ ? std::dynamic_pointer_cast(arm_->kinematicsSolver()) : nullptr; - if (pinocchio_solver) { - achieved_ok = pinocchio_solver->computeTwistBaseAtQ( - q_now, - qdot_cmd, - config_.alignment().ibvs().camera_link(), - achieved_twist_base); - } - - const auto& target_c = tracker_.lastTargetInCamera(); - const Eigen::Vector3d err_c = target_c - target_position_in_camera; - const auto& v_visp = ibvs_.lastCameraTwistVisp(); - const double qdot_norm = - qdot_cmd.empty() - ? 0.0 - : Eigen::Map( - qdot_cmd.data(), - static_cast(qdot_cmd.size())).norm(); - CMVR_LOG(DEBUG) << "[TouchScreenTask][ALIGN_DEBUG]" - << " target_c=[" << target_c.x() << ", " << target_c.y() - << ", " << target_c.z() << "]" - << ", err_c=[" << err_c.x() << ", " << err_c.y() - << ", " << err_c.z() << "]" - << ", v_visp=[" << v_visp[0] << ", " << v_visp[1] - << ", " << v_visp[2] << ", " << v_visp[3] - << ", " << v_visp[4] << ", " << v_visp[5] << "]" - << ", qdot0=" << (qdot_cmd.empty() ? 0.0 : qdot_cmd.front()) - << ", qdot_norm=" << qdot_norm - << ", achieved_ok=" << achieved_ok - << ", achieved_twist_base=[" << achieved_twist_base[0] - << ", " << achieved_twist_base[1] - << ", " << achieved_twist_base[2] - << ", " << achieved_twist_base[3] - << ", " << achieved_twist_base[4] - << ", " << achieved_twist_base[5] << "]"; + const Eigen::Matrix3d R_B_G = T_B_G.block<3, 3>(0, 0); + double T_B_G_delta_translation_m = 0.0; + double T_B_G_delta_rotation_rad = 0.0; + if (have_last_T_B_G_) { + T_B_G_delta_translation_m = + (T_B_G.block<3, 1>(0, 3) - last_T_B_G_.block<3, 1>(0, 3)).norm(); + const Eigen::Matrix3d R_delta = + last_T_B_G_.block<3, 3>(0, 0).transpose() * R_B_G; + T_B_G_delta_rotation_rad = + cmvr::device::cartesian_motion::rotationVector(R_delta).norm(); + max_T_B_G_translation_delta_m_ = + std::max(max_T_B_G_translation_delta_m_, T_B_G_delta_translation_m); + max_T_B_G_rotation_delta_rad_ = + std::max(max_T_B_G_rotation_delta_rad_, T_B_G_delta_rotation_rad); } - - if (!sendJointVelocity(qdot_cmd)) { - enterFailed(Status::ROBOT_COMMAND_FAILED); + last_T_B_G_ = T_B_G; + have_last_T_B_G_ = true; + const Eigen::Vector3d linear_B = R_B_G * output.linear_velocity_G; + const Eigen::Vector3d angular_B = R_B_G * output.angular_velocity_G; + if (!linear_B.allFinite() || !angular_B.allFinite()) { + enterFailed(Status::ALIGN_COMPUTE_FAILED); return false; } - last_align_error_camera_ = tracker_.lastTargetInCamera() - target_position_in_camera; - const Eigen::Vector3d rot_error_vec = - rotvecFromRotationMatrix(R_target.transpose() * R_current); - bool align_ok = - std::abs(last_align_error_camera_.x()) <= error_threshold[0] && - std::abs(last_align_error_camera_.y()) <= error_threshold[1] && - std::abs(last_align_error_camera_.z()) <= error_threshold[2]; - switch (alignment.target().mode()) { - case cmvr::config::TOUCH_SCREEN_ALIGN_MODE_POSE_AND_POSITION: - align_ok = align_ok && - std::abs(rot_error_vec.x()) <= error_threshold[3] && - std::abs(rot_error_vec.y()) <= error_threshold[4] && - std::abs(rot_error_vec.z()) <= error_threshold[5]; - break; - case cmvr::config::TOUCH_SCREEN_ALIGN_MODE_RX_RY_AND_POSITION: - align_ok = align_ok && - std::abs(rot_error_vec.x()) <= error_threshold[3] && - std::abs(rot_error_vec.y()) <= error_threshold[4]; - break; - case cmvr::config::TOUCH_SCREEN_ALIGN_MODE_POSITION_ONLY: - break; + const device::CartesianVelocity twist_B{ + linear_B.x(), linear_B.y(), linear_B.z(), + angular_B.x(), angular_B.y(), angular_B.z()}; + + if ((pbvs_debug_count_++ % 20) == 0) { + constexpr double kMetersToMillimeters = 1000.0; + constexpr double kRadiansToDegrees = 180.0 / 3.14159265358979323846; + const Eigen::Vector3d p_target_used_G = + pbvs_.targetPose().block<3, 1>(0, 3); + const Eigen::Vector3d p_tip_G = T_G_P.block<3, 1>(0, 3); + const Eigen::Vector3d p_B_G = T_B_G.block<3, 1>(0, 3); + const Eigen::Vector3d rotvec_B_G = + cmvr::device::cartesian_motion::rotationVector(R_B_G); + const Eigen::Matrix4d& T_C_H = tcp_pose_tracker_.T_C_H(); + const Eigen::Vector3d hand_tag_position_C = T_C_H.block<3, 1>(0, 3); + const Eigen::Vector3d hand_tag_normal_C = T_C_H.block<3, 3>(0, 0).col(2); + const double hand_tag_front_dot_camera = + hand_tag_position_C.norm() > 1e-9 + ? hand_tag_normal_C.dot(-hand_tag_position_C.normalized()) + : 0.0; + + CMVR_LOG(DEBUG) << "[PBVS_DEBUG]" + << " target_G_mm=[" + << p_target_used_G.x() * kMetersToMillimeters << ' ' + << p_target_used_G.y() * kMetersToMillimeters << ' ' + << p_target_used_G.z() * kMetersToMillimeters << ']' + << " tip_G_mm=[" + << p_tip_G.x() * kMetersToMillimeters << ' ' + << p_tip_G.y() * kMetersToMillimeters << ' ' + << p_tip_G.z() * kMetersToMillimeters << ']' + << " err_G_mm=[" + << output.position_error_G.x() * kMetersToMillimeters << ' ' + << output.position_error_G.y() * kMetersToMillimeters << ' ' + << output.position_error_G.z() * kMetersToMillimeters << ']' + << " rot_err_deg=[" + << output.rotation_error_G.x() * kRadiansToDegrees << ' ' + << output.rotation_error_G.y() * kRadiansToDegrees << ' ' + << output.rotation_error_G.z() * kRadiansToDegrees << ']' + << " raw_G=[" + << output.raw_linear_velocity_G.x() << ' ' + << output.raw_linear_velocity_G.y() << ' ' + << output.raw_linear_velocity_G.z() << ' ' + << output.raw_angular_velocity_G.x() << ' ' + << output.raw_angular_velocity_G.y() << ' ' + << output.raw_angular_velocity_G.z() << ']' + << " cmd_G=[" + << output.linear_velocity_G.x() << ' ' + << output.linear_velocity_G.y() << ' ' + << output.linear_velocity_G.z() << ' ' + << output.angular_velocity_G.x() << ' ' + << output.angular_velocity_G.y() << ' ' + << output.angular_velocity_G.z() << ']' + << " T_B_G_pos=[" + << p_B_G.x() * kMetersToMillimeters << ' ' + << p_B_G.y() * kMetersToMillimeters << ' ' + << p_B_G.z() * kMetersToMillimeters << ']' + << " T_B_G_rot=[" + << rotvec_B_G.x() * kRadiansToDegrees << ' ' + << rotvec_B_G.y() * kRadiansToDegrees << ' ' + << rotvec_B_G.z() * kRadiansToDegrees << ']' + << " T_B_G_delta_mm=" + << T_B_G_delta_translation_m * kMetersToMillimeters + << " T_B_G_delta_rot_deg=" + << T_B_G_delta_rotation_rad * kRadiansToDegrees + << " T_B_G_max_delta_mm=" + << max_T_B_G_translation_delta_m_ * kMetersToMillimeters + << " T_B_G_max_delta_rot_deg=" + << max_T_B_G_rotation_delta_rad_ * kRadiansToDegrees + << " hand_tag_front_dot_camera=" + << hand_tag_front_dot_camera + << " cmd_B=[" + << twist_B.vx << ' ' << twist_B.vy << ' ' << twist_B.vz << ' ' + << twist_B.wx << ' ' << twist_B.wy << ' ' << twist_B.wz << ']'; } - if (align_ok) { + + last_align_error_screen_tag_ = output.position_error_G; + if (output.reached) { ++align_stable_count_; } else { align_stable_count_ = 0; @@ -1159,13 +1254,13 @@ bool TouchScreenTask::stepAligning(const double dt) { if (align_stable_count_ >= alignment.stable_frames()) { CMVR_LOG(INFO) << "[TouchScreenTask][ALIGN_REACHED] tag_id=" << tag_id - << ", err_xyz=[" << last_align_error_camera_.x() << ", " - << last_align_error_camera_.y() << ", " - << last_align_error_camera_.z() << "]" - << ", err_rxyz=[" << rot_error_vec.x() << ", " - << rot_error_vec.y() << ", " - << rot_error_vec.z() << "]"; - hardStopIbvsMotion(); + << ", err_xyz_G=[" << output.position_error_G.x() << ", " + << output.position_error_G.y() << ", " + << output.position_error_G.z() << "]" + << ", err_rxyz_G=[" << output.rotation_error_G.x() << ", " + << output.rotation_error_G.y() << ", " + << output.rotation_error_G.z() << "]"; + stopPbvsMotion(); phase_ = Phase::ALIGN_REACHED; phase_start_time_ = Clock::now(); touch_command_started_ = false; @@ -1173,6 +1268,17 @@ bool TouchScreenTask::stepAligning(const double dt) { return true; } + const auto speed_result = arm_->speedL(twist_B, + pbvs_command_acceleration_, + 0.0, + device::FrameType::Base); + if (!speed_result.ok()) { + CMVR_LOG(ERROR) << "[TouchScreenTask][ALIGNING] speedL failed: " + << speed_result.message; + enterFailed(Status::ROBOT_COMMAND_FAILED); + return false; + } + last_status_ = Status::ALIGNING; return true; } @@ -1335,7 +1441,6 @@ bool TouchScreenTask::stepRetracting() { enterFailed(Status::ROBOT_COMMAND_FAILED); return false; } - holdCurrentControlledPosition(); if ((phase_after_retract_ == Phase::DONE || phase_after_retract_ == Phase::FAILED) && !moveToInitPositionIfEnabled()) { phase_ = Phase::FAILED; @@ -1347,55 +1452,14 @@ bool TouchScreenTask::stepRetracting() { return phase_ != Phase::FAILED; } -bool TouchScreenTask::readControlledJointPositions(std::vector& q_out) const { - if (!arm_) { - return false; - } - - const auto state = arm_->getJointState(); - const auto model = arm_->getRobotModel(); - std::unordered_map q_map; - q_map.reserve(model.joint_names.size()); - for (size_t i = 0; i < model.joint_names.size() && i < state.position.size(); ++i) { - q_map[model.joint_names[i]] = state.position[i]; - } - - const auto& control_joint_names = config_.alignment().ibvs().control_joint_names(); - q_out.resize(static_cast(control_joint_names.size())); - for (int i = 0; i < control_joint_names.size(); ++i) { - const auto it = q_map.find(control_joint_names[i]); - if (it == q_map.end()) { - return false; +void TouchScreenTask::stopPbvsMotion() { + if (arm_) { + try { + arm_->stopL(); + } catch (...) { } - q_out[static_cast(i)] = it->second; } - return true; -} - -bool TouchScreenTask::sendJointVelocity(const std::vector& qdot) const { - if (!arm_ || qdot.size() != static_cast( - config_.alignment().ibvs().control_joint_names_size())) { - return false; - } - - device::JointVelocityCommand cmd; - cmd.velocity = qdot; - const auto result = arm_->speedJ(cmd, 0.0, 0.0); - if (!result.ok()) { - return false; - } - return true; -} - -bool TouchScreenTask::sendZeroJointVelocity() const { - std::vector zero( - static_cast(config_.alignment().ibvs().control_joint_names_size()), 0.0); - return sendJointVelocity(zero); -} - -void TouchScreenTask::hardStopIbvsMotion() { - sendZeroJointVelocity(); - ibvs_.resetTwistCommandState(); + pbvs_.resetTwistCommandState(); } bool TouchScreenTask::readCurrentTouchPointPositionBase(Eigen::Vector3d& p_out) const { @@ -1447,39 +1511,12 @@ void TouchScreenTask::logTouchingSpeedLState() const { CMVR_LOG(DEBUG) << state_log.str(); } -bool TouchScreenTask::holdCurrentControlledPosition() const { +bool TouchScreenTask::buildInitJointPositions(std::vector& positions_out) const { if (!arm_) { return false; } - - const auto state = arm_->getJointState(); - const auto model = arm_->getRobotModel(); - std::unordered_map q_map; - q_map.reserve(model.joint_names.size()); - for (size_t i = 0; i < model.joint_names.size() && i < state.position.size(); ++i) { - q_map[model.joint_names[i]] = state.position[i]; - } - - device::JointPositionCommand joints; - joints.position.reserve(static_cast( - config_.alignment().ibvs().control_joint_names_size())); - for (const auto& name : config_.alignment().ibvs().control_joint_names()) { - const auto it = q_map.find(name); - if (it == q_map.end()) { - return false; - } - joints.position.push_back(it->second); - } - - const auto result = arm_->servoJ(joints); - if (!result.ok()) { - return false; - } - return true; -} - -bool TouchScreenTask::buildInitJointPositions(std::vector& positions_out) const { - return buildInitJointPositionsFromConfig(config_, positions_out); + return buildInitJointPositionsFromConfig( + config_, arm_->getRobotModel().joint_names, positions_out); } bool TouchScreenTask::moveToInitPositionBeforeStartIfEnabled() { @@ -1666,14 +1703,7 @@ bool TouchScreenTask::startRetractPhase(const Phase next_phase_after_retract, } void TouchScreenTask::enterFailed(const Status status) { - try { - if (arm_) { - arm_->stopL(); - } - } catch (...) { - } - hardStopIbvsMotion(); - holdCurrentControlledPosition(); + stopPbvsMotion(); phase_ = Phase::FAILED; touch_command_started_ = false; retract_command_started_ = false; diff --git a/cmvr-es/task/touch_screen_task/src/touch_screen_task_test.cpp b/cmvr-es/task/touch_screen_task/src/touch_screen_task_test.cpp index 62800082..f848a792 100644 --- a/cmvr-es/task/touch_screen_task/src/touch_screen_task_test.cpp +++ b/cmvr-es/task/touch_screen_task/src/touch_screen_task_test.cpp @@ -1,6 +1,7 @@ #include "gtest/gtest.h" #include +#include #include #include #include @@ -22,6 +23,7 @@ #include "common/vision/image_projection.h" #include "common/io/proto_file_io.h" #include "cmvr/config/arm_config/arm_config.pb.h" +#include "cmvr/config/logger_config/logger_config.pb.h" #include "cmvr/config/motor_config/motor_config.pb.h" #include "cmvr/config/touch_screen_task_config/touch_screen_task_config.pb.h" #include "devices/arm/robot_arm_factory.h" @@ -208,9 +210,9 @@ void run_touch_once(int u, int v) { << p_c_target.z() << "]" << ", nonzero_count=" << task->lastTouchNonzeroCount() << ", pressure_sum=" << task->lastTouchPressureSum() - << ", err_c=[" << task->lastAlignErrorCamera().x() << ", " - << task->lastAlignErrorCamera().y() << ", " - << task->lastAlignErrorCamera().z() << "]\n"; + << ", err_G=[" << task->lastAlignErrorScreenTag().x() << ", " + << task->lastAlignErrorScreenTag().y() << ", " + << task->lastAlignErrorScreenTag().z() << "]\n"; if (task->lastStatus() == cmvr::task::TouchScreenTask::Status::ALIGN_REACHED) { std::cout << "align reached, target_c=[" << p_c_target.x() << ", " @@ -264,9 +266,15 @@ TEST(TouchScreenTaskTest, ConfigFilesUseStructuredSchema) { const auto& config = root_config.touch_screen_task(); EXPECT_TRUE(config.has_initialization()); EXPECT_TRUE(config.has_perception()); + EXPECT_TRUE(config.perception().has_external_apriltag()); EXPECT_TRUE(config.has_alignment()); - EXPECT_TRUE(config.alignment().has_ibvs()); - EXPECT_TRUE(config.alignment().ibvs().has_camera_link()); + ASSERT_TRUE(config.alignment().has_pbvs()); + const auto& pbvs = config.alignment().pbvs(); + EXPECT_DOUBLE_EQ(pbvs.position_gain().x(), 2.0); + EXPECT_DOUBLE_EQ(pbvs.rotation_gain().z(), 1.5); + EXPECT_DOUBLE_EQ(pbvs.vmax6().z(), 0.05); + EXPECT_DOUBLE_EQ(pbvs.amax6().rz(), 2.0); + EXPECT_DOUBLE_EQ(pbvs.twist_filter_alpha(), 1.0); EXPECT_TRUE(config.has_touch()); EXPECT_EQ(config.touch().motion_case(), cmvr::config::TouchScreenTaskTouchConfig::kSpeedL); @@ -274,7 +282,322 @@ TEST(TouchScreenTaskTest, ConfigFilesUseStructuredSchema) { } } -TEST(TouchScreenTaskTest, RunTouchOnceInMujoco) { +TEST(TouchScreenTaskTest, RunEyeToHandTouchInMujoco) { + const auto project_root = findProjectRoot(); + ASSERT_FALSE(project_root.empty()); + + cmvr::config::LoggerRootConfig logger_root; + const auto logger_config_path = project_root / "cmvr-es/config/logger/logger.pb.txt"; + ASSERT_TRUE(ProtoMessageIo::getProtoFromAsciiFile( + logger_config_path.string(), &logger_root)) + << logger_config_path; + ASSERT_TRUE(cmvr::logging::initLogging( + logger_root.logger(), "touch_screen_task_test", project_root)); + + cmvr::ConfigHelper::setConfigRootFromFile( + (project_root / "cmvr-es/config/cmvr_es.pb.txt").string()); + + cmvr::task::TaskManager::destroyInstance(); + cmvr::device::DeviceManager::destroyInstance(); + + // This test intentionally builds only the devices required by the + // MuJoCo Eye-to-Hand task. The task itself is still loaded from + // touch_screen_task_mujoco.pb.txt below. + cmvr::config::DeviceManagerConfig device_manager_config; + device_manager_config.set_name("touch_screen_eye_to_hand_mujoco_test"); + device_manager_config.set_version("test"); + + auto add_device = [&](const char* id, + cmvr::config::DeviceConfigEntry::DeviceType type, + const char* config_file) { + auto* entry = device_manager_config.add_devices(); + entry->set_id(id); + entry->set_type(type); + entry->set_config_file(config_file); + entry->set_enable(true); + }; + + add_device("mujoco_world", + cmvr::config::DeviceConfigEntry::DEVICE_TYPE_MUJOCO_WORLD, + "devices/mujoco/right_arm_eye_to_hand_world.pb.txt"); + add_device("right_arm_mujoco_motors", + cmvr::config::DeviceConfigEntry::DEVICE_TYPE_MOTOR_SYSTEM, + "devices/motor/mujoco_motors.pb.txt"); + add_device("mujoco_right_arm", + cmvr::config::DeviceConfigEntry::DEVICE_TYPE_ROBOT_ARM, + "devices/arm/arm_mujoco.pb.txt"); + add_device("mujoco_zero_touch_dexhand", + cmvr::config::DeviceConfigEntry::DEVICE_TYPE_DEXHAND, + "devices/dexhand/dexhand.pb.txt"); + add_device("mujoco_hand_cam", + cmvr::config::DeviceConfigEntry::DEVICE_TYPE_CAMERA, + "devices/camera/camera.pb.txt"); + add_device("mujoco_external_touch_cam", + cmvr::config::DeviceConfigEntry::DEVICE_TYPE_CAMERA, + "devices/camera/camera.pb.txt"); + + auto& device_manager = + cmvr::device::DeviceManager::getInstance(device_manager_config); + auto arm = device_manager.getDevice("mujoco_right_arm"); + auto hand_camera = device_manager.getDevice( + "mujoco_hand_cam"); + auto external_camera = device_manager.getDevice( + "mujoco_external_touch_cam"); + ASSERT_NE(arm, nullptr); + ASSERT_NE(hand_camera, nullptr); + ASSERT_NE(external_camera, nullptr); + + const auto hand_mujoco_camera = + std::dynamic_pointer_cast(hand_camera); + const auto external_mujoco_camera = + std::dynamic_pointer_cast(external_camera); + ASSERT_NE(hand_mujoco_camera, nullptr); + ASSERT_NE(external_mujoco_camera, nullptr); + + device_manager.start(); + + cv::Mat hand_camera_frame; + cmvr::device::Rs2Intrinsics hand_camera_intrinsics{}; + hand_camera->getRGBImage(hand_camera_frame, hand_camera_intrinsics); + ASSERT_FALSE(hand_camera_frame.empty()) + << "mujoco_hand_cam did not produce a frame after DeviceManager::start()"; + + cv::Mat external_camera_frame; + cmvr::device::Rs2Intrinsics external_camera_intrinsics{}; + external_camera->getRGBImage(external_camera_frame, external_camera_intrinsics); + ASSERT_FALSE(external_camera_frame.empty()) + << "mujoco_external_touch_cam did not produce a frame after DeviceManager::start()"; + + auto world = cmvr::device::MotorManager::mujocoWorldFor( + "right_arm_mujoco_motors"); + ASSERT_NE(world, nullptr); + ASSERT_TRUE(world->isLoaded()); + ASSERT_TRUE(world->isRunning()); + + cmvr::MuJocoViewer viewer(world); + ASSERT_NE(viewer.model(), nullptr); + ASSERT_GE(mj_name2id(viewer.model(), mjOBJ_CAMERA, "hand_cam"), 0); + ASSERT_GE(mj_name2id(viewer.model(), mjOBJ_CAMERA, "external_touch_cam"), 0); + viewer.setupCamera(2.5, -160.0, -25.0); + + // PiP is display-only. Its visibility and layout follow camera.pb.txt; + // camera acquisition still uses each MujocoCamera's offscreen thread. + bool first_pip_camera = true; + const auto add_configured_pip = [&](const cmvr::device::MujocoCamera& camera) { + const auto& camera_config = camera.config(); + const auto& pip = camera_config.viewer_pip(); + if (!pip.enable()) { + return; + } + + const auto& render = camera_config.render(); + if (first_pip_camera) { + viewer.enablePiPCamera(camera_config.camera_name().c_str(), + pip.left(), + pip.bottom(), + pip.width(), + pip.height(), + render.width(), + render.height()); + first_pip_camera = false; + } else { + viewer.addPiPCamera(camera_config.camera_name().c_str(), + pip.left(), + pip.bottom(), + pip.width(), + pip.height(), + render.width(), + render.height()); + } + }; + add_configured_pip(*external_mujoco_camera); + add_configured_pip(*hand_mujoco_camera); + + struct Outcome { + bool init_ok{false}; + bool touch_ok{false}; + bool align_reached{false}; + bool retracting_seen{false}; + bool finished{false}; + cmvr::task::TouchScreenTask::Phase final_phase{ + cmvr::task::TouchScreenTask::Phase::IDLE}; + cmvr::task::TouchScreenTask::Status final_status{ + cmvr::task::TouchScreenTask::Status::IDLE}; + double max_pressure{0.0}; + double speedl_command_norm{0.0}; + int active_tag_id{-1}; + std::string error; + } outcome; + + // The viewer owns the lifetime of this test. Closing its window asks the + // worker to stop; a completed touch cycle must not close the viewer. + std::atomic stop_requested{false}; + std::thread scenario([&] { + try { + const auto touch_config = loadMujocoTouchConfig(project_root); + if (touch_config.devices().arm_id() != "mujoco_right_arm" || + touch_config.devices().camera_id() != "mujoco_hand_cam" || + touch_config.devices().external_camera_id() != + "mujoco_external_touch_cam") { + throw std::runtime_error( + "touch_screen_task_mujoco.pb.txt has unexpected device IDs"); + } + + cmvr::config::TaskManagerConfig task_manager_config; + auto* task_entry = task_manager_config.add_tasks(); + task_entry->set_id(touch_config.id()); + task_entry->set_type( + cmvr::config::TaskConfigEntry::TASK_TYPE_TOUCH_SCREEN); + task_entry->set_run_mode( + cmvr::config::TaskConfigEntry::TASK_RUN_MODE_PERIODIC_STEP); + task_entry->set_control_period_s(0.001); + task_entry->set_config_file( + "tasks/touch_screen_task/touch_screen_task_mujoco.pb.txt"); + task_entry->set_enable(true); + + auto& task_manager = + cmvr::task::TaskManager::getInstance(task_manager_config); + auto task = task_manager.getTouchScreenTask(touch_config.id()); + if (!task) { + throw std::runtime_error("TouchScreenTask not found: " + + touch_config.id()); + } + outcome.init_ok = + task->lastStatus() != + cmvr::task::TouchScreenTask::Status::NOT_INITIALIZED; + if (!outcome.init_ok) { + throw std::runtime_error("TouchScreenTask initialization failed"); + } + + task_manager.startRunTask(); + + const auto& render = hand_mujoco_camera->config().render(); + const int target_u = render.width() > 0 ? render.width() / 2 : 640; + const int target_v = render.height() > 0 ? render.height() / 2 : 360; + std::cout << "[TouchScreenTaskMujocoTest] touch target pixel: u=" + << target_u << ", v=" << target_v << std::endl; + + const bool touch_started = task->touch(target_u, target_v); + if (!touch_started) { + throw std::runtime_error( + "task->touch failed, status=" + + std::string(cmvr::task::TouchScreenTask::statusToString( + task->lastStatus()))); + } + + outcome.touch_ok = true; + std::cout << "[TouchScreenTaskMujocoTest] touch started once" + << std::endl; + + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(35); + int step_count = 0; + bool paused_at_alignment = false; + while (!stop_requested.load(std::memory_order_acquire) && + task->isBusy() && + std::chrono::steady_clock::now() < deadline) { + outcome.final_phase = task->phase(); + outcome.final_status = task->lastStatus(); + outcome.active_tag_id = task->lastActiveTagId(); + outcome.max_pressure = std::max( + outcome.max_pressure, task->lastTouchPressureSum()); + + const auto speedl = arm->getSpeedLCommandTwistBase(); + outcome.speedl_command_norm = std::max( + outcome.speedl_command_norm, + std::sqrt(speedl.vx * speedl.vx + speedl.vy * speedl.vy + + speedl.vz * speedl.vz)); + const auto phase = task->phase(); + if (task->lastStatus() == + cmvr::task::TouchScreenTask::Status::ALIGN_REACHED || + phase == cmvr::task::TouchScreenTask::Phase::TOUCHING || + phase == cmvr::task::TouchScreenTask::Phase::DWELLING || + phase == cmvr::task::TouchScreenTask::Phase::RETRACTING || + phase == cmvr::task::TouchScreenTask::Phase::DONE) { + outcome.align_reached = true; + } + if (phase == cmvr::task::TouchScreenTask::Phase::RETRACTING) { + outcome.retracting_seen = true; + } + if (phase == cmvr::task::TouchScreenTask::Phase::ALIGN_REACHED && + touch_config.alignment().pause_when_reached()) { + paused_at_alignment = true; + break; + } + if ((step_count++ % 20) == 0) { + std::cout << "[TouchScreenTaskMujocoTest] phase=" + << cmvr::task::TouchScreenTask::phaseToString( + task->phase()) + << ", status=" + << cmvr::task::TouchScreenTask::statusToString( + task->lastStatus()) + << ", active_tag=" << task->lastActiveTagId() + << ", tcp_pose=" + << cmvr::perception::TagRelativeTcpPose::statusToString( + task->tcpPoseTracker().lastStatus()) + << ", pressure=" << task->lastTouchPressureSum() + << std::endl; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + + if (stop_requested.load(std::memory_order_acquire)) { + task->stop(); + } else if (task->isBusy() && !paused_at_alignment) { + task->stop(); + throw std::runtime_error("touch scenario exceeded 35 seconds"); + } + + outcome.finished = task->isFinished(); + outcome.final_phase = task->phase(); + outcome.final_status = task->lastStatus(); + std::cout << "[TouchScreenTaskMujocoTest] touch finished once, phase=" + << cmvr::task::TouchScreenTask::phaseToString( + outcome.final_phase) + << "; viewer remains running" << std::endl; + + // Keep the viewer and its camera frames alive after the task has + // run once. The main thread sets stop_requested when the window + // is closed. + while (!stop_requested.load(std::memory_order_acquire)) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + task->stop(); + task_manager.stopRunTask(); + std::cout << "[TouchScreenTaskMujocoTest] viewer is still running; " + "close the MuJoCo window to finish gtest." << std::endl; + } catch (const std::exception& error) { + try { + cmvr::task::TaskManager::getInstance().stopRunTask(); + } catch (...) { + } + outcome.error = error.what(); + stop_requested.store(true, std::memory_order_release); + std::cerr << "[TouchScreenTaskMujocoTest] scenario error: " + << outcome.error + << "; viewer remains open, close it manually to finish gtest." + << std::endl; + } + }); + + viewer.setRunning(true); + viewer.run(); + stop_requested.store(true, std::memory_order_release); + scenario.join(); + + cmvr::task::TaskManager::destroyInstance(); + device_manager.stop(); + cmvr::device::DeviceManager::destroyInstance(); + + EXPECT_TRUE(outcome.error.empty()) << outcome.error; + EXPECT_TRUE(outcome.init_ok); + EXPECT_TRUE(outcome.touch_ok); + EXPECT_TRUE(outcome.align_reached); + EXPECT_GT(outcome.speedl_command_norm, 0.005); +} + +TEST(TouchScreenTaskTest, DISABLED_RunTouchOnceInMujoco) { const auto project_root = findProjectRoot(); ASSERT_FALSE(project_root.empty()); cmvr::ConfigHelper::setConfigRootFromFile( diff --git a/model/april_tag/tag36_11_00001.png b/model/april_tag/tag36_11_00001.png new file mode 100644 index 00000000..3d73cfb0 Binary files /dev/null and b/model/april_tag/tag36_11_00001.png differ diff --git a/model/april_tag/tag36_11_00002_1000.png b/model/april_tag/tag36_11_00002_1000.png new file mode 100644 index 00000000..6f98d5c1 Binary files /dev/null and b/model/april_tag/tag36_11_00002_1000.png differ diff --git a/model/april_tag/tag36_11_00003_1000.png b/model/april_tag/tag36_11_00003_1000.png new file mode 100644 index 00000000..cd8482ae Binary files /dev/null and b/model/april_tag/tag36_11_00003_1000.png differ diff --git a/model/april_tag/tag36_11_00004_1000.png b/model/april_tag/tag36_11_00004_1000.png new file mode 100644 index 00000000..19d95b12 Binary files /dev/null and b/model/april_tag/tag36_11_00004_1000.png differ diff --git a/model/xiaoyan_description/right_arm_eye_to_hand.xml b/model/xiaoyan_description/right_arm_eye_to_hand.xml new file mode 100644 index 00000000..a843a93a --- /dev/null +++ b/model/xiaoyan_description/right_arm_eye_to_hand.xml @@ -0,0 +1,809 @@ + + + + + + + + diff --git a/protos/cmvr/config/touch_screen_algorithm_config.proto b/protos/cmvr/config/touch_screen_algorithm_config.proto index e7e577fe..c7c72bfc 100644 --- a/protos/cmvr/config/touch_screen_algorithm_config.proto +++ b/protos/cmvr/config/touch_screen_algorithm_config.proto @@ -11,17 +11,10 @@ message TouchScreenApriltagConfig { optional TouchScreenTargetPointMethod target_point_method = 3; } -message TouchScreenIbvsConfig { - optional string camera_link = 1; - optional double lambda = 2; - optional double mu = 3; - optional double qdot_max = 4; - .cmvr.common.Vec6 vmax6 = 5; - .cmvr.common.Vec6 amax6 = 6; - optional double twist_filter_alpha = 7; - - .cmvr.common.Mat3 r_camera_to_visp = 12; - .cmvr.common.Mat3 r_camera_to_urdf = 13; - - repeated string control_joint_names = 14; +message TouchScreenTaskPbvsConfig { + .cmvr.common.Vec3 position_gain = 1; + .cmvr.common.Vec3 rotation_gain = 2; + .cmvr.common.Vec6 vmax6 = 3; + .cmvr.common.Vec6 amax6 = 4; + optional double twist_filter_alpha = 5; } diff --git a/protos/cmvr/config/touch_screen_task_config/touch_screen_task_config.proto b/protos/cmvr/config/touch_screen_task_config/touch_screen_task_config.proto index c48681a7..6dd9554d 100644 --- a/protos/cmvr/config/touch_screen_task_config/touch_screen_task_config.proto +++ b/protos/cmvr/config/touch_screen_task_config/touch_screen_task_config.proto @@ -18,6 +18,7 @@ message TouchScreenTaskDevicesConfig { optional string arm_id = 4; optional string dexhand_id = 2; optional string camera_id = 3; + optional string external_camera_id = 5; } message TouchScreenTaskInitializationConfig { @@ -28,12 +29,18 @@ message TouchScreenTaskInitializationConfig { optional double acceleration = 5; } +message TouchScreenTaskExternalApriltagConfig { + optional double tag_size_m = 1; + optional int32 hand_tag_id = 2; + .cmvr.common.Mat4 t_h_p = 3; +} + message TouchScreenTaskPerceptionConfig { TouchScreenApriltagConfig apriltag = 1; + TouchScreenTaskExternalApriltagConfig external_apriltag = 2; } message TouchScreenAlignmentTargetConfig { - .cmvr.common.Vec3 position_in_camera = 1; .cmvr.common.Vec3 rotation_vector = 2; optional TouchScreenAlignMode mode = 3; } @@ -41,12 +48,12 @@ message TouchScreenAlignmentTargetConfig { message TouchScreenTaskAlignmentConfig { reserved 1; reserved "kinematics"; - TouchScreenIbvsConfig ibvs = 2; TouchScreenAlignmentTargetConfig target = 3; .cmvr.common.Vec6 error_threshold = 4; optional int32 stable_frames = 5; optional double timeout_s = 6; optional bool pause_when_reached = 7; + TouchScreenTaskPbvsConfig pbvs = 8; } message TouchScreenTouchSpeedLConfig {