91 lines
3.2 KiB
Python
91 lines
3.2 KiB
Python
import csv
|
|
import io
|
|
import unittest
|
|
from contextlib import redirect_stdout
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
|
|
from cockpit_grounding.benchmark.reporting import (
|
|
create_run_directory,
|
|
print_model_selection,
|
|
write_root_outputs,
|
|
)
|
|
|
|
|
|
class BenchmarkReportingTest(unittest.TestCase):
|
|
def test_case_results_joins_models_by_case(self) -> None:
|
|
predictions = [
|
|
{
|
|
"model": "qwen3vl_2b",
|
|
"id": "button",
|
|
"target": "按钮",
|
|
"pred_center_pixel": [10, 20],
|
|
"total_mean_ms": 100.0,
|
|
"parse_success": True,
|
|
"pred_bbox_pixel": [1, 2, 19, 38],
|
|
"image": "/dataset/image.jpg",
|
|
},
|
|
{
|
|
"model": "qwen3vl_4b",
|
|
"id": "button",
|
|
"target": "按钮",
|
|
"pred_center_pixel": [11, 21],
|
|
"total_mean_ms": 120.0,
|
|
"parse_success": True,
|
|
"pred_bbox_pixel": [2, 3, 20, 39],
|
|
"image": "/dataset/image.jpg",
|
|
},
|
|
]
|
|
summaries = {"qwen3vl_2b": {}, "qwen3vl_4b": {}}
|
|
|
|
with TemporaryDirectory() as directory:
|
|
output = Path(directory)
|
|
write_root_outputs(output, {}, summaries, predictions)
|
|
with (output / "case_results.csv").open(
|
|
encoding="utf-8",
|
|
newline="",
|
|
) as csv_file:
|
|
rows = list(csv.DictReader(csv_file))
|
|
review = (output / "review.md").read_text(encoding="utf-8")
|
|
|
|
self.assertEqual(len(rows), 1)
|
|
self.assertEqual(rows[0]["id"], "button")
|
|
self.assertEqual(rows[0]["qwen3vl_2b_center_x"], "10")
|
|
self.assertEqual(rows[0]["qwen3vl_4b_center_y"], "21")
|
|
self.assertEqual(rows[0]["qwen3vl_2b_latency_ms"], "100.0")
|
|
self.assertEqual(rows[0]["manual_qwen3vl_2b"], "")
|
|
self.assertEqual(rows[0]["qwen3vl_4b_bbox_x2"], "20")
|
|
self.assertIn("qwen3vl_2b/visualizations/button.jpg", review)
|
|
self.assertIn("qwen3vl_4b/visualizations/button.jpg", review)
|
|
|
|
def test_exact_run_directory_uses_run_name_without_timestamp(self) -> None:
|
|
with TemporaryDirectory() as directory:
|
|
path = create_run_directory(
|
|
Path(directory),
|
|
"selection_run",
|
|
timestamped=False,
|
|
)
|
|
|
|
self.assertEqual(path.name, "selection_run")
|
|
|
|
def test_model_selection_marks_accuracy_as_manual_without_gt(self) -> None:
|
|
summary = {
|
|
"parse_success_rate": 1.0,
|
|
"mean_total_ms": 100.0,
|
|
"p50_total_ms": 99.0,
|
|
"p95_total_ms": 110.0,
|
|
"peak_cuda_memory_mb": 1000.0,
|
|
"throughput_samples_per_sec": 10.0,
|
|
}
|
|
output = io.StringIO()
|
|
with redirect_stdout(output):
|
|
print_model_selection({"model_a": summary, "model_b": summary})
|
|
|
|
printed = output.getvalue()
|
|
self.assertIn("MODEL SELECTION", printed)
|
|
self.assertIn("N/A - manual review required", printed)
|
|
accuracy_row = next(
|
|
line for line in printed.splitlines() if line.startswith("Text UI Accuracy")
|
|
)
|
|
self.assertEqual(accuracy_row.count("N/A"), 2)
|