142 lines
4.9 KiB
Python
142 lines
4.9 KiB
Python
import json
|
|
import unittest
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
|
|
from cockpit_grounding.benchmark.config import (
|
|
load_benchmark_config,
|
|
load_manifest,
|
|
)
|
|
|
|
|
|
class BenchmarkConfigTest(unittest.TestCase):
|
|
def test_load_benchmark_config_preserves_model_order(self) -> None:
|
|
with TemporaryDirectory() as directory:
|
|
tmp_path = Path(directory)
|
|
model_a = tmp_path / "model-a"
|
|
model_b = tmp_path / "model-b"
|
|
model_a.mkdir()
|
|
model_b.mkdir()
|
|
config = tmp_path / "benchmark.toml"
|
|
config.write_text(
|
|
"\n".join(
|
|
(
|
|
"[benchmark]",
|
|
"warmup = 1",
|
|
"repeats = 3",
|
|
"max_new_tokens = 128",
|
|
f'output_root = "{tmp_path}"',
|
|
"[models.first]",
|
|
f'path = "{model_a}"',
|
|
"[models.second]",
|
|
f'path = "{model_b}"',
|
|
)
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
loaded = load_benchmark_config(config)
|
|
|
|
self.assertEqual(
|
|
[model.name for model in loaded.models],
|
|
["first", "second"],
|
|
)
|
|
self.assertEqual(
|
|
[model.backend for model in loaded.models],
|
|
["qwen3vl", "qwen3vl"],
|
|
)
|
|
self.assertEqual(loaded.settings.repeats, 3)
|
|
self.assertTrue(loaded.settings.timestamp_run_directory)
|
|
|
|
def test_load_benchmark_config_supports_explicit_backend(self) -> None:
|
|
with TemporaryDirectory() as directory:
|
|
tmp_path = Path(directory)
|
|
model_path = tmp_path / "model"
|
|
model_path.mkdir()
|
|
config = tmp_path / "benchmark.toml"
|
|
config.write_text(
|
|
"\n".join(
|
|
(
|
|
"[benchmark]",
|
|
"warmup = 0",
|
|
"repeats = 1",
|
|
"max_new_tokens = 16",
|
|
f'output_root = "{tmp_path}"',
|
|
"[models.qwen35]",
|
|
'backend = "qwen35"',
|
|
f'path = "{model_path}"',
|
|
)
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
loaded = load_benchmark_config(config)
|
|
|
|
self.assertEqual(loaded.models[0].backend, "qwen35")
|
|
|
|
def test_load_benchmark_config_supports_exact_run_directory(self) -> None:
|
|
with TemporaryDirectory() as directory:
|
|
tmp_path = Path(directory)
|
|
model_path = tmp_path / "model"
|
|
model_path.mkdir()
|
|
config = tmp_path / "benchmark.toml"
|
|
config.write_text(
|
|
"\n".join(
|
|
(
|
|
"[benchmark]",
|
|
"warmup = 3",
|
|
"repeats = 5",
|
|
"max_new_tokens = 128",
|
|
f'output_root = "{tmp_path}"',
|
|
"timestamp_run_directory = false",
|
|
"[models.model]",
|
|
f'path = "{model_path}"',
|
|
)
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
loaded = load_benchmark_config(config)
|
|
|
|
self.assertFalse(loaded.settings.timestamp_run_directory)
|
|
|
|
def test_manifest_supports_optional_gt(self) -> None:
|
|
with TemporaryDirectory() as directory:
|
|
tmp_path = Path(directory)
|
|
image = tmp_path / "image.jpg"
|
|
image.touch()
|
|
manifest = tmp_path / "samples.jsonl"
|
|
entries = (
|
|
{"id": "without-gt", "image": str(image), "target": "button"},
|
|
{
|
|
"id": "with-gt",
|
|
"image": str(image),
|
|
"target": "button",
|
|
"gt_bbox_pixel": [1, 2, 3, 4],
|
|
},
|
|
)
|
|
manifest.write_text(
|
|
"\n".join(json.dumps(item) for item in entries) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
samples = load_manifest(manifest)
|
|
|
|
self.assertIsNone(samples[0].gt_bbox_pixel)
|
|
self.assertEqual(samples[1].gt_bbox_pixel, (1.0, 2.0, 3.0, 4.0))
|
|
|
|
def test_manifest_rejects_duplicate_ids(self) -> None:
|
|
with TemporaryDirectory() as directory:
|
|
tmp_path = Path(directory)
|
|
image = tmp_path / "image.jpg"
|
|
image.touch()
|
|
manifest = tmp_path / "samples.jsonl"
|
|
item = {"id": "same", "image": str(image), "target": "button"}
|
|
manifest.write_text(
|
|
json.dumps(item) + "\n" + json.dumps(item) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with self.assertRaisesRegex(ValueError, "Duplicate"):
|
|
load_manifest(manifest)
|