514 lines
16 KiB
Python
514 lines
16 KiB
Python
|
from __future__ import annotations
|
||
|
|
|
||
|
|
from collections.abc import Mapping
|
||
|
|
from copy import deepcopy
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from cmvr_edge_ai.compiler import PipelineCompileError, compile_pipeline
|
||
|
|
from cmvr_edge_ai.config import load_config_data
|
||
|
|
from cmvr_edge_ai.core import Envelope, Operator, Sink, Source
|
||
|
|
from cmvr_edge_ai.plugins import (
|
||
|
|
InvocationCardinality,
|
||
|
|
PluginKind,
|
||
|
|
PluginRegistry,
|
||
|
|
PluginSpec,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
class EmptySource(Source):
|
||
|
|
async def messages(self): # type: ignore[no-untyped-def]
|
||
|
|
if False:
|
||
|
|
yield Envelope("unused")
|
||
|
|
|
||
|
|
|
||
|
|
class IdentityOperator(Operator):
|
||
|
|
async def process(
|
||
|
|
self, envelope: Envelope[Any], input_port: str = "input"
|
||
|
|
) -> Envelope[Any]:
|
||
|
|
del input_port
|
||
|
|
return envelope
|
||
|
|
|
||
|
|
|
||
|
|
class NullSink(Sink):
|
||
|
|
async def consume(
|
||
|
|
self, envelope: Envelope[Any], input_port: str = "input"
|
||
|
|
) -> None:
|
||
|
|
del envelope, input_port
|
||
|
|
|
||
|
|
|
||
|
|
def _source_factory(node_id: str, params: Mapping[str, Any]) -> EmptySource:
|
||
|
|
del node_id, params
|
||
|
|
return EmptySource()
|
||
|
|
|
||
|
|
|
||
|
|
def _operator_factory(
|
||
|
|
node_id: str, params: Mapping[str, Any]
|
||
|
|
) -> IdentityOperator:
|
||
|
|
del node_id, params
|
||
|
|
return IdentityOperator()
|
||
|
|
|
||
|
|
|
||
|
|
def _sink_factory(node_id: str, params: Mapping[str, Any]) -> NullSink:
|
||
|
|
del node_id, params
|
||
|
|
return NullSink()
|
||
|
|
|
||
|
|
|
||
|
|
def _registry() -> PluginRegistry:
|
||
|
|
registry = PluginRegistry()
|
||
|
|
registry.register(
|
||
|
|
PluginSpec(
|
||
|
|
"server.request_source@1",
|
||
|
|
PluginKind.SOURCE,
|
||
|
|
_source_factory,
|
||
|
|
outputs={"requests": "InferenceRequest/v1"},
|
||
|
|
)
|
||
|
|
)
|
||
|
|
registry.register(
|
||
|
|
PluginSpec(
|
||
|
|
"test.image_decoder@1",
|
||
|
|
PluginKind.OPERATOR,
|
||
|
|
_operator_factory,
|
||
|
|
inputs={"requests": "InferenceRequest/v1"},
|
||
|
|
outputs={"frames": "ImageFrame/v1"},
|
||
|
|
invocation_cardinality=InvocationCardinality.EXACTLY_ONE,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
registry.register(
|
||
|
|
PluginSpec(
|
||
|
|
"test.frame_identity@1",
|
||
|
|
PluginKind.OPERATOR,
|
||
|
|
_operator_factory,
|
||
|
|
inputs={"frames": "ImageFrame/v1"},
|
||
|
|
outputs={"frames": "ImageFrame/v1"},
|
||
|
|
invocation_cardinality=InvocationCardinality.EXACTLY_ONE,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
registry.register(
|
||
|
|
PluginSpec(
|
||
|
|
"test.unsafe_identity@1",
|
||
|
|
PluginKind.OPERATOR,
|
||
|
|
_operator_factory,
|
||
|
|
inputs={"requests": "InferenceRequest/v1"},
|
||
|
|
outputs={"frames": "ImageFrame/v1"},
|
||
|
|
)
|
||
|
|
)
|
||
|
|
registry.register(
|
||
|
|
PluginSpec(
|
||
|
|
"detection.model@1",
|
||
|
|
PluginKind.OPERATOR,
|
||
|
|
_operator_factory,
|
||
|
|
inputs={"frames": "ImageFrame/v1"},
|
||
|
|
outputs={"detections": "DetectionResult/v1"},
|
||
|
|
route_model_param="model",
|
||
|
|
invocation_cardinality=InvocationCardinality.EXACTLY_ONE,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
registry.register(
|
||
|
|
PluginSpec(
|
||
|
|
"test.generic_model@1",
|
||
|
|
PluginKind.OPERATOR,
|
||
|
|
_operator_factory,
|
||
|
|
inputs={"requests": "InferenceRequest/v1"},
|
||
|
|
outputs={"responses": "InferenceResponse/v1"},
|
||
|
|
route_model_param="model_id",
|
||
|
|
invocation_cardinality=InvocationCardinality.EXACTLY_ONE,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
registry.register(
|
||
|
|
PluginSpec(
|
||
|
|
"test.detector_bypass@1",
|
||
|
|
PluginKind.OPERATOR,
|
||
|
|
_operator_factory,
|
||
|
|
inputs={"frames": "ImageFrame/v1"},
|
||
|
|
outputs={"detections": "DetectionResult/v1"},
|
||
|
|
invocation_cardinality=InvocationCardinality.EXACTLY_ONE,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
registry.register(
|
||
|
|
PluginSpec(
|
||
|
|
"server.detection_response@1",
|
||
|
|
PluginKind.OPERATOR,
|
||
|
|
_operator_factory,
|
||
|
|
inputs={"detections": "DetectionResult/v1"},
|
||
|
|
outputs={"responses": "InferenceResponse/v1"},
|
||
|
|
invocation_cardinality=InvocationCardinality.EXACTLY_ONE,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
registry.register(
|
||
|
|
PluginSpec(
|
||
|
|
"test.untyped_response@1",
|
||
|
|
PluginKind.OPERATOR,
|
||
|
|
_operator_factory,
|
||
|
|
inputs={"detections": "DetectionResult/v1"},
|
||
|
|
outputs={"responses": "*"},
|
||
|
|
invocation_cardinality=InvocationCardinality.EXACTLY_ONE,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
registry.register(
|
||
|
|
PluginSpec(
|
||
|
|
"server.response_sink@1",
|
||
|
|
PluginKind.SINK,
|
||
|
|
_sink_factory,
|
||
|
|
inputs={"responses": "InferenceResponse/v1"},
|
||
|
|
)
|
||
|
|
)
|
||
|
|
registry.register(
|
||
|
|
PluginSpec(
|
||
|
|
"test.active_source@1",
|
||
|
|
PluginKind.SOURCE,
|
||
|
|
_source_factory,
|
||
|
|
outputs={"events": "TextEvent/v1"},
|
||
|
|
)
|
||
|
|
)
|
||
|
|
registry.register(
|
||
|
|
PluginSpec(
|
||
|
|
"test.active_sink@1",
|
||
|
|
PluginKind.SINK,
|
||
|
|
_sink_factory,
|
||
|
|
inputs={"events": "TextEvent/v1"},
|
||
|
|
)
|
||
|
|
)
|
||
|
|
return registry
|
||
|
|
|
||
|
|
|
||
|
|
def _valid_nodes() -> dict[str, dict[str, Any]]:
|
||
|
|
return {
|
||
|
|
"request": {
|
||
|
|
"uses": "server.request_source@1",
|
||
|
|
"with": {"category": "detect.mobile_phone"},
|
||
|
|
},
|
||
|
|
"decode": {"uses": "test.image_decoder@1"},
|
||
|
|
"detector": {
|
||
|
|
"uses": "detection.model@1",
|
||
|
|
"with": {"model": "mobile-phone@1"},
|
||
|
|
},
|
||
|
|
"response": {
|
||
|
|
"uses": "server.detection_response@1",
|
||
|
|
"with": {"category": "detect.mobile_phone"},
|
||
|
|
},
|
||
|
|
"sink": {"uses": "server.response_sink@1"},
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def _valid_edges() -> list[dict[str, Any]]:
|
||
|
|
return [
|
||
|
|
{"from": "request.requests", "to": "decode.requests"},
|
||
|
|
{"from": "decode.frames", "to": "detector.frames"},
|
||
|
|
{"from": "detector.detections", "to": "response.detections"},
|
||
|
|
{"from": "response.responses", "to": "sink.responses"},
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
def _config(
|
||
|
|
*,
|
||
|
|
nodes: dict[str, dict[str, Any]] | None = None,
|
||
|
|
edges: list[dict[str, str]] | None = None,
|
||
|
|
): # type: ignore[no-untyped-def]
|
||
|
|
return load_config_data(
|
||
|
|
{
|
||
|
|
"api_version": "cmvr.edge.ai/v1",
|
||
|
|
"pipelines": {
|
||
|
|
"remote": {
|
||
|
|
"nodes": deepcopy(nodes if nodes is not None else _valid_nodes()),
|
||
|
|
"edges": deepcopy(edges if edges is not None else _valid_edges()),
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"server": {
|
||
|
|
"enabled": True,
|
||
|
|
"routes": {
|
||
|
|
"detect.mobile_phone": {
|
||
|
|
"pipeline": "remote",
|
||
|
|
"model_id": "mobile-phone@1",
|
||
|
|
}
|
||
|
|
},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_compile_accepts_strict_detection_invocation_pipeline() -> None:
|
||
|
|
compiled = compile_pipeline(_config(), "remote", _registry())
|
||
|
|
|
||
|
|
assert compiled.pipeline_id == "remote"
|
||
|
|
assert set(compiled.plugin_specs) == set(_valid_nodes())
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
("boundary", "message"),
|
||
|
|
[
|
||
|
|
("request", "exactly one server.request_source@1"),
|
||
|
|
("sink", "exactly one server.response_sink@1"),
|
||
|
|
],
|
||
|
|
)
|
||
|
|
def test_compile_requires_one_request_and_response_boundary(
|
||
|
|
boundary: str, message: str
|
||
|
|
) -> None:
|
||
|
|
nodes = _valid_nodes()
|
||
|
|
del nodes[boundary]
|
||
|
|
edges = [
|
||
|
|
edge
|
||
|
|
for edge in _valid_edges()
|
||
|
|
if not edge["from"].startswith(boundary + ".")
|
||
|
|
and not edge["to"].startswith(boundary + ".")
|
||
|
|
]
|
||
|
|
|
||
|
|
with pytest.raises(PipelineCompileError, match=message):
|
||
|
|
compile_pipeline(_config(nodes=nodes, edges=edges), "remote", _registry())
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
("original", "duplicate", "message"),
|
||
|
|
[
|
||
|
|
("request", "request_copy", "exactly one server.request_source@1"),
|
||
|
|
("sink", "sink_copy", "exactly one server.response_sink@1"),
|
||
|
|
],
|
||
|
|
)
|
||
|
|
def test_compile_rejects_duplicate_invocation_boundaries(
|
||
|
|
original: str, duplicate: str, message: str
|
||
|
|
) -> None:
|
||
|
|
nodes = _valid_nodes()
|
||
|
|
nodes[duplicate] = deepcopy(nodes[original])
|
||
|
|
|
||
|
|
with pytest.raises(PipelineCompileError, match=message):
|
||
|
|
compile_pipeline(_config(nodes=nodes), "remote", _registry())
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("node_id", ["request", "response"])
|
||
|
|
def test_compile_requires_source_and_response_categories_to_match_route(
|
||
|
|
node_id: str,
|
||
|
|
) -> None:
|
||
|
|
nodes = _valid_nodes()
|
||
|
|
nodes[node_id]["with"]["category"] = "detect.other" # type: ignore[index]
|
||
|
|
|
||
|
|
with pytest.raises(PipelineCompileError, match="does not match route category"):
|
||
|
|
compile_pipeline(_config(nodes=nodes), "remote", _registry())
|
||
|
|
|
||
|
|
|
||
|
|
def test_compile_requires_exactly_one_registered_detection_node() -> None:
|
||
|
|
missing = _valid_nodes()
|
||
|
|
missing["detector"]["uses"] = "test.detector_bypass@1"
|
||
|
|
with pytest.raises(PipelineCompileError, match="exactly one detection.model@1"):
|
||
|
|
compile_pipeline(_config(nodes=missing), "remote", _registry())
|
||
|
|
|
||
|
|
duplicate = _valid_nodes()
|
||
|
|
duplicate["detector_copy"] = deepcopy(duplicate["detector"])
|
||
|
|
with pytest.raises(PipelineCompileError, match="exactly one detection.model@1"):
|
||
|
|
compile_pipeline(_config(nodes=duplicate), "remote", _registry())
|
||
|
|
|
||
|
|
|
||
|
|
def test_compile_requires_detector_model_to_match_route() -> None:
|
||
|
|
nodes = _valid_nodes()
|
||
|
|
nodes["detector"]["with"]["model"] = "different@1" # type: ignore[index]
|
||
|
|
|
||
|
|
with pytest.raises(PipelineCompileError, match="does not match route model_id"):
|
||
|
|
compile_pipeline(_config(nodes=nodes), "remote", _registry())
|
||
|
|
|
||
|
|
|
||
|
|
def test_compile_rejects_rate_limited_passive_detector() -> None:
|
||
|
|
nodes = _valid_nodes()
|
||
|
|
nodes["detector"]["with"]["max_fps"] = 1.0 # type: ignore[index]
|
||
|
|
|
||
|
|
with pytest.raises(PipelineCompileError, match="must not set max_fps"):
|
||
|
|
compile_pipeline(_config(nodes=nodes), "remote", _registry())
|
||
|
|
|
||
|
|
|
||
|
|
def test_compile_requires_exactly_one_output_contract_for_each_operator() -> None:
|
||
|
|
nodes = _valid_nodes()
|
||
|
|
nodes["decode"]["uses"] = "test.unsafe_identity@1"
|
||
|
|
|
||
|
|
with pytest.raises(PipelineCompileError, match="invocation_cardinality"):
|
||
|
|
compile_pipeline(_config(nodes=nodes), "remote", _registry())
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
("profile", "overflow"),
|
||
|
|
[
|
||
|
|
("telemetry", "drop_oldest"),
|
||
|
|
("request", "reject"),
|
||
|
|
],
|
||
|
|
)
|
||
|
|
def test_compile_requires_lossless_blocking_invocation_edges(
|
||
|
|
profile: str,
|
||
|
|
overflow: str,
|
||
|
|
) -> None:
|
||
|
|
edges: list[dict[str, Any]] = deepcopy(_valid_edges())
|
||
|
|
edges[0]["qos"] = {"profile": profile, "overflow": overflow}
|
||
|
|
|
||
|
|
with pytest.raises(PipelineCompileError, match="must use qos"):
|
||
|
|
compile_pipeline(_config(edges=edges), "remote", _registry())
|
||
|
|
|
||
|
|
|
||
|
|
def test_compile_validates_route_model_for_non_detection_adapter() -> None:
|
||
|
|
def config(model_id: str): # type: ignore[no-untyped-def]
|
||
|
|
return load_config_data(
|
||
|
|
{
|
||
|
|
"api_version": "cmvr.edge.ai/v1",
|
||
|
|
"pipelines": {
|
||
|
|
"gauge": {
|
||
|
|
"nodes": {
|
||
|
|
"request": {
|
||
|
|
"uses": "server.request_source@1",
|
||
|
|
"with": {"category": "gauge.analog"},
|
||
|
|
},
|
||
|
|
"reader": {
|
||
|
|
"uses": "test.generic_model@1",
|
||
|
|
"with": {
|
||
|
|
"category": "gauge.analog",
|
||
|
|
"model_id": model_id,
|
||
|
|
},
|
||
|
|
},
|
||
|
|
"sink": {"uses": "server.response_sink@1"},
|
||
|
|
},
|
||
|
|
"edges": [
|
||
|
|
{
|
||
|
|
"from": "request.requests",
|
||
|
|
"to": "reader.requests",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"from": "reader.responses",
|
||
|
|
"to": "sink.responses",
|
||
|
|
},
|
||
|
|
],
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"server": {
|
||
|
|
"enabled": True,
|
||
|
|
"routes": {
|
||
|
|
"gauge.analog": {
|
||
|
|
"pipeline": "gauge",
|
||
|
|
"model_id": "ethz-gauge@1",
|
||
|
|
}
|
||
|
|
},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
assert compile_pipeline(config("ethz-gauge@1"), "gauge", _registry())
|
||
|
|
with pytest.raises(PipelineCompileError, match="route model_id"):
|
||
|
|
compile_pipeline(config("another-gauge@1"), "gauge", _registry())
|
||
|
|
|
||
|
|
|
||
|
|
def test_compile_rejects_fan_out_and_merge_without_join_semantics() -> None:
|
||
|
|
nodes = _valid_nodes()
|
||
|
|
nodes["left"] = {"uses": "test.frame_identity@1"}
|
||
|
|
nodes["right"] = {"uses": "test.frame_identity@1"}
|
||
|
|
edges = [
|
||
|
|
{"from": "request.requests", "to": "decode.requests"},
|
||
|
|
{"from": "decode.frames", "to": "left.frames"},
|
||
|
|
{"from": "decode.frames", "to": "right.frames"},
|
||
|
|
{"from": "left.frames", "to": "detector.frames"},
|
||
|
|
{"from": "right.frames", "to": "detector.frames"},
|
||
|
|
{"from": "detector.detections", "to": "response.detections"},
|
||
|
|
{"from": "response.responses", "to": "sink.responses"},
|
||
|
|
]
|
||
|
|
|
||
|
|
with pytest.raises(PipelineCompileError, match="single linear path"):
|
||
|
|
compile_pipeline(_config(nodes=nodes, edges=edges), "remote", _registry())
|
||
|
|
|
||
|
|
|
||
|
|
def test_compile_requires_a_directed_request_to_response_path() -> None:
|
||
|
|
edges = _valid_edges()[:-1]
|
||
|
|
|
||
|
|
with pytest.raises(PipelineCompileError, match="no directed path"):
|
||
|
|
compile_pipeline(_config(edges=edges), "remote", _registry())
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("connect_branch", [False, True])
|
||
|
|
def test_compile_rejects_orphan_and_dead_end_branch_nodes(
|
||
|
|
connect_branch: bool,
|
||
|
|
) -> None:
|
||
|
|
nodes = _valid_nodes()
|
||
|
|
nodes["unused"] = {"uses": "test.frame_identity@1"}
|
||
|
|
edges = _valid_edges()
|
||
|
|
if connect_branch:
|
||
|
|
edges.append({"from": "decode.frames", "to": "unused.frames"})
|
||
|
|
|
||
|
|
with pytest.raises(PipelineCompileError, match="outside every valid"):
|
||
|
|
compile_pipeline(
|
||
|
|
_config(nodes=nodes, edges=edges),
|
||
|
|
"remote",
|
||
|
|
_registry(),
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_compile_rejects_path_that_bypasses_required_detector() -> None:
|
||
|
|
nodes = _valid_nodes()
|
||
|
|
nodes["bypass"] = {"uses": "test.detector_bypass@1"}
|
||
|
|
edges = _valid_edges()
|
||
|
|
edges.extend(
|
||
|
|
[
|
||
|
|
{"from": "decode.frames", "to": "bypass.frames"},
|
||
|
|
{"from": "bypass.detections", "to": "response.detections"},
|
||
|
|
]
|
||
|
|
)
|
||
|
|
|
||
|
|
with pytest.raises(PipelineCompileError, match="bypasses required detection"):
|
||
|
|
compile_pipeline(
|
||
|
|
_config(nodes=nodes, edges=edges),
|
||
|
|
"remote",
|
||
|
|
_registry(),
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_compile_requires_one_typed_response_producer() -> None:
|
||
|
|
nodes = _valid_nodes()
|
||
|
|
nodes["response"]["uses"] = "test.untyped_response@1"
|
||
|
|
|
||
|
|
with pytest.raises(PipelineCompileError, match="emits InferenceResponse/v1"):
|
||
|
|
compile_pipeline(_config(nodes=nodes), "remote", _registry())
|
||
|
|
|
||
|
|
|
||
|
|
def test_active_push_pipeline_is_not_subject_to_invocation_validation() -> None:
|
||
|
|
config = load_config_data(
|
||
|
|
{
|
||
|
|
"api_version": "cmvr.edge.ai/v1",
|
||
|
|
"pipelines": {
|
||
|
|
"active": {
|
||
|
|
"nodes": {
|
||
|
|
"source": {"uses": "test.active_source@1"},
|
||
|
|
"sink": {"uses": "test.active_sink@1"},
|
||
|
|
},
|
||
|
|
"edges": [
|
||
|
|
{"from": "source.events", "to": "sink.events"},
|
||
|
|
],
|
||
|
|
}
|
||
|
|
},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
compiled = compile_pipeline(config, "active", _registry())
|
||
|
|
|
||
|
|
assert compiled.pipeline_id == "active"
|
||
|
|
|
||
|
|
|
||
|
|
def test_disabled_server_routes_do_not_change_active_pipeline_validation() -> None:
|
||
|
|
config = load_config_data(
|
||
|
|
{
|
||
|
|
"api_version": "cmvr.edge.ai/v1",
|
||
|
|
"pipelines": {
|
||
|
|
"active": {
|
||
|
|
"nodes": {
|
||
|
|
"source": {"uses": "test.active_source@1"},
|
||
|
|
"sink": {"uses": "test.active_sink@1"},
|
||
|
|
},
|
||
|
|
"edges": [
|
||
|
|
{"from": "source.events", "to": "sink.events"},
|
||
|
|
],
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"server": {
|
||
|
|
"enabled": False,
|
||
|
|
"routes": {
|
||
|
|
"detect.mobile_phone": {
|
||
|
|
"pipeline": "active",
|
||
|
|
"model_id": "mobile-phone@1",
|
||
|
|
}
|
||
|
|
},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
assert compile_pipeline(config, "active", _registry()).pipeline_id == "active"
|