import unittest from cockpit_agent.common.enums import ( Domain, Operation, SelectionState, Unit, Zone, ) from cockpit_agent.grounding.adapter import GroundingResult from cockpit_agent.intent.schemas import SemanticAction from cockpit_agent.perception.schemas import UIState from cockpit_agent.perception.ui_understanding import ModelUIUnderstanding class FakeVisionGenerator: def __init__(self, output: str) -> None: self.output = output self.prompt = "" self.image_path = "" def generate( self, image_path: str, prompt: str, max_new_tokens: int = 256, ) -> str: del max_new_tokens self.image_path = image_path self.prompt = prompt return self.output def temperature_intent() -> SemanticAction: return SemanticAction( domain=Domain.CLIMATE, function="temperature", zone=Zone.DRIVER, operation=Operation.SET, state=None, value=23, unit=Unit.CELSIUS, ) def function_grounding(query: str = "主驾驶温度控制") -> GroundingResult: return GroundingResult( semantic_query=query, bbox_relative=(100, 200, 400, 600), bbox_pixel=(20, 20, 80, 60), center_pixel=(50, 40), image_width=200, image_height=100, raw_output='{"bbox_2d": [100, 200, 400, 600]}', ) def understand(model: FakeVisionGenerator, intent: SemanticAction): return ModelUIUnderstanding(model).understand( "/tmp/function_roi.jpg", intent, function_grounding(), ) class UIUnderstandingTest(unittest.TestCase): def test_slider_schema_allows_nullable_observations(self) -> None: state = UIState.from_dict( { "function": "tailgate_max_opening", "zone": None, "control_type": "slider", "current_value": None, "step_value": None, "current_state": None, "selection_state": None, "min_value": None, "max_value": None, "options": None, "track_visible": True, "knob_visible": None, "orientation": None, } ) self.assertIsNone(state.current_value) self.assertIsNone(state.min_value) self.assertIsNone(state.max_value) self.assertTrue(state.track_visible) self.assertIsNone(state.knob_visible) def test_selection_state_schema_accepts_selected_and_unknown(self) -> None: for selection_state in ("selected", "unknown"): with self.subTest(selection_state=selection_state): state = UIState.from_dict( { "function": "unlock_mode", "zone": None, "control_type": "segmented_control", "current_value": "all_doors", "step_value": None, "current_state": None, "selection_state": selection_state, "min_value": None, "max_value": None, "options": ["driver_door", "all_doors"], "track_visible": None, "knob_visible": None, "orientation": None, } ) self.assertEqual(state.selection_state.value, selection_state) def test_partial_options_do_not_invalidate_observed_selection(self) -> None: state = UIState.from_dict( { "function": "unlock_mode", "zone": None, "control_type": "segmented_control", "current_value": "all_doors", "step_value": None, "current_state": None, "selection_state": "selected", "min_value": None, "max_value": None, "options": ["driver_door"], "track_visible": None, "knob_visible": None, "orientation": None, } ) self.assertEqual(state.current_value, "all_doors") self.assertEqual(state.options, ("driver_door",)) self.assertEqual(state.selection_state, SelectionState.SELECTED) def test_clears_fields_that_do_not_apply_to_stepper(self) -> None: model = FakeVisionGenerator( """{ "control_type": "stepper", "current_value": 26.0, "step_value": 1.0, "current_state": null, "min_value": 26.0, "max_value": 26.0, "options": null }""" ) state = understand(model, temperature_intent()) self.assertEqual(state.current_value, 26.0) self.assertEqual(state.function, "temperature") self.assertEqual(state.zone, Zone.DRIVER) self.assertIsNone(state.min_value) self.assertIsNone(state.max_value) def test_slider_allows_unobservable_current_value(self) -> None: intent = SemanticAction.from_dict( { "domain": "body", "function": "tailgate_max_opening", "zone": None, "operation": "set", "state": None, "value": 80, "unit": "percent", } ) model = FakeVisionGenerator( """{ "control_type": "slider", "current_value": null, "step_value": null, "current_state": null, "min_value": "50%", "max_value": "100%", "options": null, "selection_state": null, "track_visible": true, "knob_visible": true ,"orientation": "horizontal" }""" ) state = understand(model, intent) self.assertIsNone(state.current_value) self.assertEqual(state.min_value, 50) self.assertEqual(state.max_value, 100) self.assertTrue(state.track_visible) self.assertTrue(state.knob_visible) self.assertEqual(state.orientation.value, "horizontal") self.assertIsNone(state.selection_state) def test_normalizes_unlock_option_labels(self) -> None: intent = SemanticAction.from_dict( { "domain": "body", "function": "unlock_mode", "zone": None, "operation": "set", "state": None, "value": "all_doors", "unit": None, } ) model = FakeVisionGenerator( """{ "control_type": "segmented_control", "current_value": "所有车门", "step_value": null, "current_state": "active", "min_value": "主驾车门", "max_value": "所有车门", "options": ["主驾车门", "所有车门"] }""" ) state = understand(model, intent) self.assertEqual(state.current_value, "all_doors") self.assertEqual(state.options, ("driver_door", "all_doors")) self.assertIsNone(state.current_state) self.assertEqual(state.selection_state, SelectionState.SELECTED) def test_normalizes_common_selection_state_language(self) -> None: intent = SemanticAction.from_dict( { "domain": "body", "function": "unlock_mode", "zone": None, "operation": "set", "state": None, "value": "all_doors", "unit": None, } ) cases = ( ("active", SelectionState.SELECTED), ("inactive", SelectionState.UNSELECTED), ("selected", SelectionState.SELECTED), ("unknown", SelectionState.UNKNOWN), ) for raw_state, expected in cases: with self.subTest(raw_state=raw_state): model = FakeVisionGenerator( """{ "control_type": "segmented_control", "current_value": "所有车门", "step_value": null, "current_state": null, "selection_state": "%s", "min_value": null, "max_value": null, "options": ["主驾车门", "所有车门"], "track_visible": null, "knob_visible": null }""" % raw_state ) state = understand(model, intent) self.assertEqual(state.selection_state, expected) def test_model_control_type_is_not_repaired_for_unlock_mode(self) -> None: intent = SemanticAction.from_dict( { "domain": "body", "function": "unlock_mode", "zone": None, "operation": "set", "state": None, "value": "all_doors", "unit": None, } ) model = FakeVisionGenerator( """{ "control_type": "button", "current_value": "所有车门", "step_value": null, "current_state": "active", "min_value": null, "max_value": null, "options": null }""" ) state = understand(model, intent) self.assertEqual(state.control_type.value, "button") self.assertEqual(state.current_value, "all_doors") self.assertEqual(state.selection_state, SelectionState.SELECTED) self.assertIsNone(state.current_state) def test_preserves_model_control_type_without_function_specific_repair(self) -> None: model = FakeVisionGenerator( """{ "control_type": "slider", "current_value": null, "step_value": null, "current_state": null, "min_value": 0, "max_value": 100, "options": null }""" ) state = understand(model, temperature_intent()) self.assertEqual(state.control_type.value, "slider") def test_all_observation_fields_may_be_null(self) -> None: model = FakeVisionGenerator( """{ "control_type": null, "current_value": null, "step_value": null, "current_state": null, "min_value": null, "max_value": null, "options": null }""" ) state = understand(model, temperature_intent()) self.assertIsNone(state.control_type) self.assertIsNone(state.current_value) self.assertIsNone(state.current_state) self.assertIsNone(state.selection_state) self.assertIsNone(state.track_visible) self.assertIsNone(state.knob_visible) def test_prompt_describes_roi_and_forbids_guessing(self) -> None: model = FakeVisionGenerator( """{ "control_type": "stepper", "current_value": 26, "step_value": 1, "current_state": null, "min_value": null, "max_value": null, "options": null }""" ) understand(model, temperature_intent()) self.assertEqual(model.image_path, "/tmp/function_roi.jpg") self.assertIn("ROI", model.prompt) self.assertIn("绝不能", model.prompt) self.assertIn("端点旁的数值", model.prompt) self.assertIn("selection_state", model.prompt) self.assertNotIn("available", model.prompt) self.assertNotIn("空调控制栏", model.prompt) self.assertNotIn("页面中央", model.prompt) if __name__ == "__main__": unittest.main()