cockpit-ui-grounding/tests/test_benchmark_config.py

import json
import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from cockpit_grounding.benchmark.config import (
    load_benchmark_config,
    load_manifest,
)


class BenchmarkConfigTest(unittest.TestCase):
    def test_load_benchmark_config_preserves_model_order(self) -> None:
        with TemporaryDirectory() as directory:
            tmp_path = Path(directory)
            model_a = tmp_path / "model-a"
            model_b = tmp_path / "model-b"
            model_a.mkdir()
            model_b.mkdir()
            config = tmp_path / "benchmark.toml"
            config.write_text(
                "\n".join(
                    (
                        "[benchmark]",
                        "warmup = 1",
                        "repeats = 3",
                        "max_new_tokens = 128",
                        f'output_root = "{tmp_path}"',
                        "[models.first]",
                        f'path = "{model_a}"',
                        "[models.second]",
                        f'path = "{model_b}"',
                    )
                ),
                encoding="utf-8",
            )

            loaded = load_benchmark_config(config)

            self.assertEqual(
                [model.name for model in loaded.models],
                ["first", "second"],
            )
            self.assertEqual(
                [model.backend for model in loaded.models],
                ["qwen3vl", "qwen3vl"],
            )
            self.assertEqual(loaded.settings.repeats, 3)
            self.assertTrue(loaded.settings.timestamp_run_directory)

    def test_load_benchmark_config_supports_explicit_backend(self) -> None:
        with TemporaryDirectory() as directory:
            tmp_path = Path(directory)
            model_path = tmp_path / "model"
            model_path.mkdir()
            config = tmp_path / "benchmark.toml"
            config.write_text(
                "\n".join(
                    (
                        "[benchmark]",
                        "warmup = 0",
                        "repeats = 1",
                        "max_new_tokens = 16",
                        f'output_root = "{tmp_path}"',
                        "[models.qwen35]",
                        'backend = "qwen35"',
                        f'path = "{model_path}"',
                    )
                ),
                encoding="utf-8",
            )

            loaded = load_benchmark_config(config)

            self.assertEqual(loaded.models[0].backend, "qwen35")

    def test_load_benchmark_config_supports_exact_run_directory(self) -> None:
        with TemporaryDirectory() as directory:
            tmp_path = Path(directory)
            model_path = tmp_path / "model"
            model_path.mkdir()
            config = tmp_path / "benchmark.toml"
            config.write_text(
                "\n".join(
                    (
                        "[benchmark]",
                        "warmup = 3",
                        "repeats = 5",
                        "max_new_tokens = 128",
                        f'output_root = "{tmp_path}"',
                        "timestamp_run_directory = false",
                        "[models.model]",
                        f'path = "{model_path}"',
                    )
                ),
                encoding="utf-8",
            )

            loaded = load_benchmark_config(config)

            self.assertFalse(loaded.settings.timestamp_run_directory)

    def test_manifest_supports_optional_gt(self) -> None:
        with TemporaryDirectory() as directory:
            tmp_path = Path(directory)
            image = tmp_path / "image.jpg"
            image.touch()
            manifest = tmp_path / "samples.jsonl"
            entries = (
                {"id": "without-gt", "image": str(image), "target": "button"},
                {
                    "id": "with-gt",
                    "image": str(image),
                    "target": "button",
                    "gt_bbox_pixel": [1, 2, 3, 4],
                },
            )
            manifest.write_text(
                "\n".join(json.dumps(item) for item in entries) + "\n",
                encoding="utf-8",
            )

            samples = load_manifest(manifest)

            self.assertIsNone(samples[0].gt_bbox_pixel)
            self.assertEqual(samples[1].gt_bbox_pixel, (1.0, 2.0, 3.0, 4.0))

    def test_manifest_rejects_duplicate_ids(self) -> None:
        with TemporaryDirectory() as directory:
            tmp_path = Path(directory)
            image = tmp_path / "image.jpg"
            image.touch()
            manifest = tmp_path / "samples.jsonl"
            item = {"id": "same", "image": str(image), "target": "button"}
            manifest.write_text(
                json.dumps(item) + "\n" + json.dumps(item) + "\n",
                encoding="utf-8",
            )

            with self.assertRaisesRegex(ValueError, "Duplicate"):
                load_manifest(manifest)