345 lines
12 KiB
Python
345 lines
12 KiB
Python
|
import unittest
|
||
|
|
|
||
|
|
from cockpit_agent.common.enums import (
|
||
|
|
Domain,
|
||
|
|
Operation,
|
||
|
|
SelectionState,
|
||
|
|
Unit,
|
||
|
|
Zone,
|
||
|
|
)
|
||
|
|
from cockpit_agent.grounding.adapter import GroundingResult
|
||
|
|
from cockpit_agent.intent.schemas import SemanticAction
|
||
|
|
from cockpit_agent.perception.schemas import UIState
|
||
|
|
from cockpit_agent.perception.ui_understanding import ModelUIUnderstanding
|
||
|
|
|
||
|
|
|
||
|
|
class FakeVisionGenerator:
|
||
|
|
def __init__(self, output: str) -> None:
|
||
|
|
self.output = output
|
||
|
|
self.prompt = ""
|
||
|
|
self.image_path = ""
|
||
|
|
|
||
|
|
def generate(
|
||
|
|
self,
|
||
|
|
image_path: str,
|
||
|
|
prompt: str,
|
||
|
|
max_new_tokens: int = 256,
|
||
|
|
) -> str:
|
||
|
|
del max_new_tokens
|
||
|
|
self.image_path = image_path
|
||
|
|
self.prompt = prompt
|
||
|
|
return self.output
|
||
|
|
|
||
|
|
|
||
|
|
def temperature_intent() -> SemanticAction:
|
||
|
|
return SemanticAction(
|
||
|
|
domain=Domain.CLIMATE,
|
||
|
|
function="temperature",
|
||
|
|
zone=Zone.DRIVER,
|
||
|
|
operation=Operation.SET,
|
||
|
|
state=None,
|
||
|
|
value=23,
|
||
|
|
unit=Unit.CELSIUS,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def function_grounding(query: str = "主驾驶温度控制") -> GroundingResult:
|
||
|
|
return GroundingResult(
|
||
|
|
semantic_query=query,
|
||
|
|
bbox_relative=(100, 200, 400, 600),
|
||
|
|
bbox_pixel=(20, 20, 80, 60),
|
||
|
|
center_pixel=(50, 40),
|
||
|
|
image_width=200,
|
||
|
|
image_height=100,
|
||
|
|
raw_output='{"bbox_2d": [100, 200, 400, 600]}',
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def understand(model: FakeVisionGenerator, intent: SemanticAction):
|
||
|
|
return ModelUIUnderstanding(model).understand(
|
||
|
|
"/tmp/function_roi.jpg",
|
||
|
|
intent,
|
||
|
|
function_grounding(),
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
class UIUnderstandingTest(unittest.TestCase):
|
||
|
|
def test_slider_schema_allows_nullable_observations(self) -> None:
|
||
|
|
state = UIState.from_dict(
|
||
|
|
{
|
||
|
|
"function": "tailgate_max_opening",
|
||
|
|
"zone": None,
|
||
|
|
"control_type": "slider",
|
||
|
|
"current_value": None,
|
||
|
|
"step_value": None,
|
||
|
|
"current_state": None,
|
||
|
|
"selection_state": None,
|
||
|
|
"min_value": None,
|
||
|
|
"max_value": None,
|
||
|
|
"options": None,
|
||
|
|
"track_visible": True,
|
||
|
|
"knob_visible": None,
|
||
|
|
"orientation": None,
|
||
|
|
}
|
||
|
|
)
|
||
|
|
self.assertIsNone(state.current_value)
|
||
|
|
self.assertIsNone(state.min_value)
|
||
|
|
self.assertIsNone(state.max_value)
|
||
|
|
self.assertTrue(state.track_visible)
|
||
|
|
self.assertIsNone(state.knob_visible)
|
||
|
|
|
||
|
|
def test_selection_state_schema_accepts_selected_and_unknown(self) -> None:
|
||
|
|
for selection_state in ("selected", "unknown"):
|
||
|
|
with self.subTest(selection_state=selection_state):
|
||
|
|
state = UIState.from_dict(
|
||
|
|
{
|
||
|
|
"function": "unlock_mode",
|
||
|
|
"zone": None,
|
||
|
|
"control_type": "segmented_control",
|
||
|
|
"current_value": "all_doors",
|
||
|
|
"step_value": None,
|
||
|
|
"current_state": None,
|
||
|
|
"selection_state": selection_state,
|
||
|
|
"min_value": None,
|
||
|
|
"max_value": None,
|
||
|
|
"options": ["driver_door", "all_doors"],
|
||
|
|
"track_visible": None,
|
||
|
|
"knob_visible": None,
|
||
|
|
"orientation": None,
|
||
|
|
}
|
||
|
|
)
|
||
|
|
self.assertEqual(state.selection_state.value, selection_state)
|
||
|
|
|
||
|
|
def test_partial_options_do_not_invalidate_observed_selection(self) -> None:
|
||
|
|
state = UIState.from_dict(
|
||
|
|
{
|
||
|
|
"function": "unlock_mode",
|
||
|
|
"zone": None,
|
||
|
|
"control_type": "segmented_control",
|
||
|
|
"current_value": "all_doors",
|
||
|
|
"step_value": None,
|
||
|
|
"current_state": None,
|
||
|
|
"selection_state": "selected",
|
||
|
|
"min_value": None,
|
||
|
|
"max_value": None,
|
||
|
|
"options": ["driver_door"],
|
||
|
|
"track_visible": None,
|
||
|
|
"knob_visible": None,
|
||
|
|
"orientation": None,
|
||
|
|
}
|
||
|
|
)
|
||
|
|
self.assertEqual(state.current_value, "all_doors")
|
||
|
|
self.assertEqual(state.options, ("driver_door",))
|
||
|
|
self.assertEqual(state.selection_state, SelectionState.SELECTED)
|
||
|
|
|
||
|
|
def test_clears_fields_that_do_not_apply_to_stepper(self) -> None:
|
||
|
|
model = FakeVisionGenerator(
|
||
|
|
"""{
|
||
|
|
"control_type": "stepper",
|
||
|
|
"current_value": 26.0,
|
||
|
|
"step_value": 1.0,
|
||
|
|
"current_state": null,
|
||
|
|
"min_value": 26.0,
|
||
|
|
"max_value": 26.0,
|
||
|
|
"options": null
|
||
|
|
}"""
|
||
|
|
)
|
||
|
|
state = understand(model, temperature_intent())
|
||
|
|
self.assertEqual(state.current_value, 26.0)
|
||
|
|
self.assertEqual(state.function, "temperature")
|
||
|
|
self.assertEqual(state.zone, Zone.DRIVER)
|
||
|
|
self.assertIsNone(state.min_value)
|
||
|
|
self.assertIsNone(state.max_value)
|
||
|
|
|
||
|
|
def test_slider_allows_unobservable_current_value(self) -> None:
|
||
|
|
intent = SemanticAction.from_dict(
|
||
|
|
{
|
||
|
|
"domain": "body",
|
||
|
|
"function": "tailgate_max_opening",
|
||
|
|
"zone": None,
|
||
|
|
"operation": "set",
|
||
|
|
"state": None,
|
||
|
|
"value": 80,
|
||
|
|
"unit": "percent",
|
||
|
|
}
|
||
|
|
)
|
||
|
|
model = FakeVisionGenerator(
|
||
|
|
"""{
|
||
|
|
"control_type": "slider",
|
||
|
|
"current_value": null,
|
||
|
|
"step_value": null,
|
||
|
|
"current_state": null,
|
||
|
|
"min_value": "50%",
|
||
|
|
"max_value": "100%",
|
||
|
|
"options": null,
|
||
|
|
"selection_state": null,
|
||
|
|
"track_visible": true,
|
||
|
|
"knob_visible": true
|
||
|
|
,"orientation": "horizontal"
|
||
|
|
}"""
|
||
|
|
)
|
||
|
|
state = understand(model, intent)
|
||
|
|
self.assertIsNone(state.current_value)
|
||
|
|
self.assertEqual(state.min_value, 50)
|
||
|
|
self.assertEqual(state.max_value, 100)
|
||
|
|
self.assertTrue(state.track_visible)
|
||
|
|
self.assertTrue(state.knob_visible)
|
||
|
|
self.assertEqual(state.orientation.value, "horizontal")
|
||
|
|
self.assertIsNone(state.selection_state)
|
||
|
|
|
||
|
|
def test_normalizes_unlock_option_labels(self) -> None:
|
||
|
|
intent = SemanticAction.from_dict(
|
||
|
|
{
|
||
|
|
"domain": "body",
|
||
|
|
"function": "unlock_mode",
|
||
|
|
"zone": None,
|
||
|
|
"operation": "set",
|
||
|
|
"state": None,
|
||
|
|
"value": "all_doors",
|
||
|
|
"unit": None,
|
||
|
|
}
|
||
|
|
)
|
||
|
|
model = FakeVisionGenerator(
|
||
|
|
"""{
|
||
|
|
"control_type": "segmented_control",
|
||
|
|
"current_value": "所有车门",
|
||
|
|
"step_value": null,
|
||
|
|
"current_state": "active",
|
||
|
|
"min_value": "主驾车门",
|
||
|
|
"max_value": "所有车门",
|
||
|
|
"options": ["主驾车门", "所有车门"]
|
||
|
|
}"""
|
||
|
|
)
|
||
|
|
state = understand(model, intent)
|
||
|
|
self.assertEqual(state.current_value, "all_doors")
|
||
|
|
self.assertEqual(state.options, ("driver_door", "all_doors"))
|
||
|
|
self.assertIsNone(state.current_state)
|
||
|
|
self.assertEqual(state.selection_state, SelectionState.SELECTED)
|
||
|
|
|
||
|
|
def test_normalizes_common_selection_state_language(self) -> None:
|
||
|
|
intent = SemanticAction.from_dict(
|
||
|
|
{
|
||
|
|
"domain": "body",
|
||
|
|
"function": "unlock_mode",
|
||
|
|
"zone": None,
|
||
|
|
"operation": "set",
|
||
|
|
"state": None,
|
||
|
|
"value": "all_doors",
|
||
|
|
"unit": None,
|
||
|
|
}
|
||
|
|
)
|
||
|
|
cases = (
|
||
|
|
("active", SelectionState.SELECTED),
|
||
|
|
("inactive", SelectionState.UNSELECTED),
|
||
|
|
("selected", SelectionState.SELECTED),
|
||
|
|
("unknown", SelectionState.UNKNOWN),
|
||
|
|
)
|
||
|
|
for raw_state, expected in cases:
|
||
|
|
with self.subTest(raw_state=raw_state):
|
||
|
|
model = FakeVisionGenerator(
|
||
|
|
"""{
|
||
|
|
"control_type": "segmented_control",
|
||
|
|
"current_value": "所有车门",
|
||
|
|
"step_value": null,
|
||
|
|
"current_state": null,
|
||
|
|
"selection_state": "%s",
|
||
|
|
"min_value": null,
|
||
|
|
"max_value": null,
|
||
|
|
"options": ["主驾车门", "所有车门"],
|
||
|
|
"track_visible": null,
|
||
|
|
"knob_visible": null
|
||
|
|
}""" % raw_state
|
||
|
|
)
|
||
|
|
state = understand(model, intent)
|
||
|
|
self.assertEqual(state.selection_state, expected)
|
||
|
|
|
||
|
|
def test_model_control_type_is_not_repaired_for_unlock_mode(self) -> None:
|
||
|
|
intent = SemanticAction.from_dict(
|
||
|
|
{
|
||
|
|
"domain": "body",
|
||
|
|
"function": "unlock_mode",
|
||
|
|
"zone": None,
|
||
|
|
"operation": "set",
|
||
|
|
"state": None,
|
||
|
|
"value": "all_doors",
|
||
|
|
"unit": None,
|
||
|
|
}
|
||
|
|
)
|
||
|
|
model = FakeVisionGenerator(
|
||
|
|
"""{
|
||
|
|
"control_type": "button",
|
||
|
|
"current_value": "所有车门",
|
||
|
|
"step_value": null,
|
||
|
|
"current_state": "active",
|
||
|
|
"min_value": null,
|
||
|
|
"max_value": null,
|
||
|
|
"options": null
|
||
|
|
}"""
|
||
|
|
)
|
||
|
|
state = understand(model, intent)
|
||
|
|
self.assertEqual(state.control_type.value, "button")
|
||
|
|
self.assertEqual(state.current_value, "all_doors")
|
||
|
|
self.assertEqual(state.selection_state, SelectionState.SELECTED)
|
||
|
|
self.assertIsNone(state.current_state)
|
||
|
|
|
||
|
|
def test_preserves_model_control_type_without_function_specific_repair(self) -> None:
|
||
|
|
model = FakeVisionGenerator(
|
||
|
|
"""{
|
||
|
|
"control_type": "slider",
|
||
|
|
"current_value": null,
|
||
|
|
"step_value": null,
|
||
|
|
"current_state": null,
|
||
|
|
"min_value": 0,
|
||
|
|
"max_value": 100,
|
||
|
|
"options": null
|
||
|
|
}"""
|
||
|
|
)
|
||
|
|
state = understand(model, temperature_intent())
|
||
|
|
self.assertEqual(state.control_type.value, "slider")
|
||
|
|
|
||
|
|
def test_all_observation_fields_may_be_null(self) -> None:
|
||
|
|
model = FakeVisionGenerator(
|
||
|
|
"""{
|
||
|
|
"control_type": null,
|
||
|
|
"current_value": null,
|
||
|
|
"step_value": null,
|
||
|
|
"current_state": null,
|
||
|
|
"min_value": null,
|
||
|
|
"max_value": null,
|
||
|
|
"options": null
|
||
|
|
}"""
|
||
|
|
)
|
||
|
|
state = understand(model, temperature_intent())
|
||
|
|
self.assertIsNone(state.control_type)
|
||
|
|
self.assertIsNone(state.current_value)
|
||
|
|
self.assertIsNone(state.current_state)
|
||
|
|
self.assertIsNone(state.selection_state)
|
||
|
|
self.assertIsNone(state.track_visible)
|
||
|
|
self.assertIsNone(state.knob_visible)
|
||
|
|
|
||
|
|
def test_prompt_describes_roi_and_forbids_guessing(self) -> None:
|
||
|
|
model = FakeVisionGenerator(
|
||
|
|
"""{
|
||
|
|
"control_type": "stepper",
|
||
|
|
"current_value": 26,
|
||
|
|
"step_value": 1,
|
||
|
|
"current_state": null,
|
||
|
|
"min_value": null,
|
||
|
|
"max_value": null,
|
||
|
|
"options": null
|
||
|
|
}"""
|
||
|
|
)
|
||
|
|
understand(model, temperature_intent())
|
||
|
|
self.assertEqual(model.image_path, "/tmp/function_roi.jpg")
|
||
|
|
self.assertIn("ROI", model.prompt)
|
||
|
|
self.assertIn("绝不能", model.prompt)
|
||
|
|
self.assertIn("端点旁的数值", model.prompt)
|
||
|
|
self.assertIn("selection_state", model.prompt)
|
||
|
|
self.assertNotIn("available", model.prompt)
|
||
|
|
self.assertNotIn("空调控制栏", model.prompt)
|
||
|
|
self.assertNotIn("页面中央", model.prompt)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
unittest.main()
|