From c8f5ae91da1c55e01a24b64ea597139f533aef68 Mon Sep 17 00:00:00 2001 From: Haran Rajkumar Date: Wed, 5 Aug 2026 13:35:01 -0700 Subject: [PATCH] feat(cli): add `adk eval --num_runs N` with per-eval-case aggregation **Problem:** `adk eval` did not support repeated runs, so users had to script external loops to reduce nondeterminism. **Solution:** - Add `--num_runs` to `adk eval` (default `1`, min `1`). The value is carried on `InferenceConfig.num_runs`, so the eval service repeats each eval case through its existing parallelism pool instead of the CLI multiplying the requests. - Aggregate the per-run results for each eval case (`eval_set_id` + `eval_id`) with a mean-over-invocations strategy that matches `AgentEvaluator` (the pytest entrypoint), so multi-run summaries are consistent across entrypoints. The strategy is an input via `EvaluateConfig.aggregation_strategy` (`AggregationStrategy`); only `MEAN_OVER_INVOCATIONS` is supported today. - `--print_detailed_results` continues to show per-run details. Tests cover the aggregator (mean-over-invocations, grouping, threshold, and runs that failed before producing metric results), the service-level `num_runs` repetition, and the CLI validation/summary behavior. Co-authored-by: ftnext Co-authored-by: Haran Rajkumar PiperOrigin-RevId: 959841339 --- src/google/adk/cli/cli_tools_click.py | 27 ++- .../_eval_case_result_aggregator.py | 157 +++++++++++++ .../adk/evaluation/base_eval_service.py | 25 +++ .../adk/evaluation/local_eval_service.py | 10 +- .../cli/utils/test_cli_tools_click.py | 196 ++++++++++++++++ .../test_eval_case_result_aggregator.py | 210 ++++++++++++++++++ .../evaluation/test_local_eval_service.py | 36 +++ 7 files changed, 657 insertions(+), 4 deletions(-) create mode 100644 src/google/adk/evaluation/_eval_case_result_aggregator.py create mode 100644 tests/unittests/evaluation/test_eval_case_result_aggregator.py diff --git a/src/google/adk/cli/cli_tools_click.py b/src/google/adk/cli/cli_tools_click.py index f14c74f9b9d..b3b372b5950 100644 --- a/src/google/adk/cli/cli_tools_click.py +++ b/src/google/adk/cli/cli_tools_click.py @@ -1142,6 +1142,13 @@ def _resolve_eval_config_file_path( ) @click.argument("eval_set_file_path_or_id", nargs=-1) @click.option("--config_file_path", help="Optional. The path to config file.") +@click.option( + "--num_runs", + type=click.IntRange(min=1), + default=1, + show_default=True, + help="Optional. Number of times each eval set should be run.", +) @click.option( "--print_detailed_results", is_flag=True, @@ -1157,6 +1164,7 @@ def cli_eval( print_detailed_results: bool, eval_storage_uri: str | None = None, log_level: str = "INFO", + num_runs: int = 1, ): """Evaluates an agent given the eval sets. @@ -1210,6 +1218,8 @@ def cli_eval( CONFIG_FILE_PATH: The path to config file. + NUM_RUNS: Number of times each eval set should be run. + PRINT_DETAILED_RESULTS: Prints detailed results on the console. """ envs.load_dotenv_for_agent(agent_module_file_path, ".") @@ -1218,6 +1228,8 @@ def cli_eval( try: import importlib # noqa: F401 + from ..evaluation._eval_case_result_aggregator import aggregate_eval_case_results + from ..evaluation.base_eval_service import EvaluateConfig from ..evaluation.base_eval_service import InferenceConfig from ..evaluation.base_eval_service import InferenceRequest from ..evaluation.eval_config import get_eval_metrics_from_config @@ -1273,9 +1285,10 @@ def cli_eval( inference_config = InferenceConfig( use_live=True, live_timeout_seconds=eval_config.live_model_config.timeout_seconds, + num_runs=num_runs, ) else: - inference_config = InferenceConfig(use_live=False) + inference_config = InferenceConfig(use_live=False, num_runs=num_runs) # Check if the first entry is a file that exists, if it does then we assume # rest of the entries are also files. We enforce this assumption in the if @@ -1350,11 +1363,15 @@ def cli_eval( app=app, ) + # `num_runs` is carried on the InferenceConfig, so the eval service repeats + # each eval case through its own parallelism pool. inference_results = asyncio.run( _collect_inferences( - inference_requests=inference_requests, eval_service=eval_service + inference_requests=inference_requests, + eval_service=eval_service, ) ) + evaluate_config = EvaluateConfig(eval_metrics=eval_metrics) eval_results = asyncio.run( _collect_eval_results( inference_results=inference_results, @@ -1362,6 +1379,10 @@ def cli_eval( eval_metrics=eval_metrics, ) ) + aggregate_eval_results = aggregate_eval_case_results( + eval_results, + aggregation_strategy=evaluate_config.aggregation_strategy, + ) except ModuleNotFoundError as mnf: raise click.ClickException(_missing_eval_dependencies_message()) from mnf @@ -1370,7 +1391,7 @@ def cli_eval( ) eval_run_summary = {} - for eval_result in eval_results: + for eval_result in aggregate_eval_results: if eval_result.eval_set_id not in eval_run_summary: eval_run_summary[eval_result.eval_set_id] = [0, 0] diff --git a/src/google/adk/evaluation/_eval_case_result_aggregator.py b/src/google/adk/evaluation/_eval_case_result_aggregator.py new file mode 100644 index 00000000000..a4b4486c1b7 --- /dev/null +++ b/src/google/adk/evaluation/_eval_case_result_aggregator.py @@ -0,0 +1,157 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Aggregation of per-run EvalCaseResults into a single result per eval case.""" + +from __future__ import annotations + +import statistics + +from .base_eval_service import AggregationStrategy +from .eval_metrics import _get_metric_threshold +from .eval_metrics import EvalMetricResult +from .eval_result import EvalCaseResult +from .evaluator import EvalStatus + + +def _generate_final_eval_status( + overall_eval_metric_results: list[EvalMetricResult], +) -> EvalStatus: + """Returns the final eval status for a case from its overall metric results.""" + final_eval_status = EvalStatus.NOT_EVALUATED + for overall_eval_metric_result in overall_eval_metric_results: + overall_eval_status = overall_eval_metric_result.eval_status + if overall_eval_status == EvalStatus.PASSED: + final_eval_status = EvalStatus.PASSED + elif overall_eval_status == EvalStatus.NOT_EVALUATED: + continue + elif overall_eval_status == EvalStatus.FAILED: + return EvalStatus.FAILED + else: + raise ValueError(f"Unknown eval status: {overall_eval_status}.") + return final_eval_status + + +def _has_hard_failed_run(per_case_results: list[EvalCaseResult]) -> bool: + """Returns True if any run failed without producing metric results. + + A run whose inferencing raised is recorded as FAILED with no per-invocation + metric results. Such a run contributes no scores to the mean, so it must be + honored explicitly or a crashed run would be silently dropped from the + aggregate verdict. Mirrors + `AgentEvaluator._get_failures_from_final_eval_status`. + """ + for result in per_case_results: + if result.final_eval_status != EvalStatus.FAILED: + continue + if not any( + invocation.eval_metric_results + for invocation in result.eval_metric_result_per_invocation + ): + return True + return False + + +def _mean_over_invocations( + per_case_results: list[EvalCaseResult], +) -> list[EvalMetricResult]: + """Returns overall metric results pooled over every invocation of every run. + + Every per-invocation metric result across all runs is pooled by metric name + and its scores are averaged into a single overall score for that metric. This + matches `AgentEvaluator`, whereas averaging each run's already-overall score + (mean-of-means) would weight runs equally regardless of invocation count. + """ + # Pool per-invocation metric results across all runs, keyed by metric name, + # preserving first-seen order (which is metric evaluation order). + results_by_metric: dict[str, list[EvalMetricResult]] = {} + for result in per_case_results: + for invocation in result.eval_metric_result_per_invocation: + for metric_result in invocation.eval_metric_results: + results_by_metric.setdefault(metric_result.metric_name, []).append( + metric_result + ) + + overall_eval_metric_results: list[EvalMetricResult] = [] + for metric_results in results_by_metric.values(): + aggregate_metric_result = metric_results[0].model_copy(deep=True) + scores = [m.score for m in metric_results if m.score is not None] + if scores: + aggregate_metric_result.score = statistics.mean(scores) + aggregate_metric_result.eval_status = ( + EvalStatus.PASSED + if aggregate_metric_result.score + >= _get_metric_threshold(aggregate_metric_result) + else EvalStatus.FAILED + ) + else: + aggregate_metric_result.score = None + aggregate_metric_result.eval_status = EvalStatus.NOT_EVALUATED + overall_eval_metric_results.append(aggregate_metric_result) + + return overall_eval_metric_results + + +def aggregate_eval_case_results( + eval_case_results: list[EvalCaseResult], + aggregation_strategy: AggregationStrategy = ( + AggregationStrategy.MEAN_OVER_INVOCATIONS + ), +) -> list[EvalCaseResult]: + """Aggregates per-run EvalCaseResults into one result per eval case. + + Results are grouped by (eval_set_id, eval_id); each group holds the runs for a + single eval case. The returned list has one EvalCaseResult per case, sorted by + (eval_set_id, eval_id). A case run once is returned unchanged apart from + grouping. + + Args: + eval_case_results: The per-run results to aggregate. + aggregation_strategy: How to combine per-run results. Only + `MEAN_OVER_INVOCATIONS` is currently supported. + """ + if aggregation_strategy != AggregationStrategy.MEAN_OVER_INVOCATIONS: + raise ValueError( + f"Unsupported aggregation strategy: {aggregation_strategy}." + ) + + results_by_case: dict[tuple[str, str], list[EvalCaseResult]] = {} + for result in eval_case_results: + key = (result.eval_set_id, result.eval_id) + results_by_case.setdefault(key, []).append(result) + + aggregate_results: list[EvalCaseResult] = [] + for per_case_results in results_by_case.values(): + overall_eval_metric_results = _mean_over_invocations(per_case_results) + final_eval_status = _generate_final_eval_status(overall_eval_metric_results) + + # A run that crashed before producing any metric results contributes no + # scores to the mean above, so honor its failure explicitly. + if final_eval_status != EvalStatus.FAILED and _has_hard_failed_run( + per_case_results + ): + final_eval_status = EvalStatus.FAILED + + aggregate_result = per_case_results[0].model_copy(deep=True) + aggregate_result.overall_eval_metric_results = overall_eval_metric_results + aggregate_result.final_eval_status = final_eval_status + # Retain every run's invocations so detailed inspection still has them. + aggregate_result.eval_metric_result_per_invocation = [ + invocation + for result in per_case_results + for invocation in result.eval_metric_result_per_invocation + ] + aggregate_results.append(aggregate_result) + + return sorted(aggregate_results, key=lambda x: (x.eval_set_id, x.eval_id)) diff --git a/src/google/adk/evaluation/base_eval_service.py b/src/google/adk/evaluation/base_eval_service.py index 34c5fe2fe55..4b04f05c807 100644 --- a/src/google/adk/evaluation/base_eval_service.py +++ b/src/google/adk/evaluation/base_eval_service.py @@ -31,6 +31,17 @@ from .eval_result import EvalCaseResult +class AggregationStrategy(Enum): + """Strategy for aggregating EvalCaseResults of the same case across runs.""" + + MEAN_OVER_INVOCATIONS = "mean_over_invocations" + """Pools per-invocation metric scores across all runs and takes their mean. + + This matches the aggregation performed by `AgentEvaluator` (the pytest eval + entrypoint), keeping multi-run summaries consistent across entrypoints. + """ + + class EvaluateConfig(BaseModel): """Contains configurations needed to run evaluations.""" @@ -54,6 +65,12 @@ class EvaluateConfig(BaseModel): """, ) + aggregation_strategy: AggregationStrategy = Field( + default=AggregationStrategy.MEAN_OVER_INVOCATIONS, + description="""Strategy used to aggregate the per-run EvalCaseResults of an +eval case when it is run more than once (see `InferenceConfig.num_runs`).""", + ) + class InferenceConfig(BaseModel): """Contains configurations need to run inferences.""" @@ -69,6 +86,14 @@ class InferenceConfig(BaseModel): charges.""", ) + num_runs: int = Field( + default=1, + ge=1, + description="""Number of times each eval case should be run. Values greater +than 1 reduce nondeterminism: the eval case is inferenced `num_runs` times +(through the same parallelism pool) and the per-run results are aggregated.""", + ) + parallelism: int = Field( default=4, description="""Number of parallel inferences to run during an Eval. Few diff --git a/src/google/adk/evaluation/local_eval_service.py b/src/google/adk/evaluation/local_eval_service.py index 16dcc514c88..880c760fe12 100644 --- a/src/google/adk/evaluation/local_eval_service.py +++ b/src/google/adk/evaluation/local_eval_service.py @@ -198,7 +198,15 @@ async def run_inference(eval_case: EvalCase) -> InferenceResult: live_timeout_seconds=inference_request.inference_config.live_timeout_seconds, ) - inference_results = [run_inference(eval_case) for eval_case in eval_cases] + # Each eval case is inferenced `num_runs` times. Running the repeats here + # (rather than in the caller) lets the parallelism semaphore above cover the + # repeated runs as well. + num_runs = inference_request.inference_config.num_runs + inference_results = [ + run_inference(eval_case) + for eval_case in eval_cases + for _ in range(num_runs) + ] for inference_result in asyncio.as_completed(inference_results): yield await inference_result diff --git a/tests/unittests/cli/utils/test_cli_tools_click.py b/tests/unittests/cli/utils/test_cli_tools_click.py index dddda4471fb..97961383b8c 100644 --- a/tests/unittests/cli/utils/test_cli_tools_click.py +++ b/tests/unittests/cli/utils/test_cli_tools_click.py @@ -38,10 +38,16 @@ from google.adk.cli import cli_tools_click from google.adk.cli.utils import gcp_utils from google.adk.evaluation.eval_case import EvalCase +from google.adk.evaluation.eval_case import Invocation +from google.adk.evaluation.eval_metrics import EvalMetricResult +from google.adk.evaluation.eval_metrics import EvalMetricResultPerInvocation +from google.adk.evaluation.eval_metrics import EvalStatus +from google.adk.evaluation.eval_result import EvalCaseResult from google.adk.evaluation.eval_set import EvalSet from google.adk.evaluation.local_eval_set_results_manager import LocalEvalSetResultsManager from google.adk.evaluation.local_eval_sets_manager import LocalEvalSetsManager from google.adk.events.event import Event +from google.genai import types as genai_types from pydantic import BaseModel import pytest @@ -1540,6 +1546,23 @@ def _fake_import(name: str, globals=None, locals=None, fromlist=(), level=0): assert MISSING_EVAL_DEPENDENCIES_MESSAGE in result.output +def test_cli_eval_rejects_num_runs_less_than_one(tmp_path: Path) -> None: + agent_dir = tmp_path / "agent_num_runs_validation" + agent_dir.mkdir() + (agent_dir / "__init__.py").touch() + eval_file = tmp_path / "dummy.evalset.json" + eval_file.touch() + + runner = CliRunner() + result = runner.invoke( + cli_tools_click.main, + ["eval", str(agent_dir), str(eval_file), "--num_runs", "0"], + ) + + assert result.exit_code != 0 + assert "Invalid value for '--num_runs'" in result.output + + # cli web & api_server (uvicorn patched) @pytest.fixture() def _patch_uvicorn(monkeypatch: pytest.MonkeyPatch) -> _Recorder: @@ -1788,6 +1811,179 @@ def test_cli_eval_with_eval_set_id( assert len(eval_set_results) == 1 +@pytest.mark.unmute_click +def test_cli_eval_with_num_runs_aggregates_per_eval_case( + mock_get_root_agent, + monkeypatch: pytest.MonkeyPatch, + tmp_path, +): + app_name = "test_app_num_runs" + eval_set_id = "test_eval_set_num_runs" + agent_path = tmp_path / app_name + agent_path.mkdir() + (agent_path / "__init__.py").touch() + + eval_sets_manager = LocalEvalSetsManager(agents_dir=str(tmp_path)) + eval_sets_manager.create_eval_set(app_name=app_name, eval_set_id=eval_set_id) + eval_sets_manager.add_eval_case( + app_name=app_name, + eval_set_id=eval_set_id, + eval_case=EvalCase(eval_id="case1", conversation=[]), + ) + + observed_num_runs = {"value": None} + + async def _fake_collect_inferences(inference_requests, eval_service): + del eval_service + # `num_runs` is now carried on the InferenceConfig; the eval service (not + # the CLI) is responsible for repeating the runs. + observed_num_runs["value"] = [ + request.inference_config.num_runs for request in inference_requests + ] + return [] + + def _run_result(score: float, session_id: str) -> EvalCaseResult: + metric_result = EvalMetricResult( + metric_name="response_match_score", + threshold=0.8, + score=score, + eval_status=(EvalStatus.PASSED if score >= 0.8 else EvalStatus.FAILED), + ) + return EvalCaseResult( + eval_set_id=eval_set_id, + eval_id="case1", + final_eval_status=metric_result.eval_status, + overall_eval_metric_results=[metric_result], + eval_metric_result_per_invocation=[ + EvalMetricResultPerInvocation( + actual_invocation=Invocation( + user_content=genai_types.Content( + parts=[genai_types.Part(text="hi")] + ) + ), + eval_metric_results=[metric_result], + ) + ], + session_id=session_id, + ) + + async def _fake_collect_eval_results( + inference_results, eval_service, eval_metrics + ): + del inference_results, eval_service, eval_metrics + # Two per-run results for the same case: mean over invocations is + # (1.0 + 0.6) / 2 = 0.8, which meets the 0.8 threshold. + return [_run_result(1.0, "session_1"), _run_result(0.6, "session_2")] + + monkeypatch.setattr( + "google.adk.cli.cli_eval._collect_inferences", _fake_collect_inferences + ) + monkeypatch.setattr( + "google.adk.cli.cli_eval._collect_eval_results", + _fake_collect_eval_results, + ) + + result = CliRunner().invoke( + cli_tools_click.main, + ["eval", str(agent_path), eval_set_id, "--num_runs", "2"], + ) + + assert result.exit_code == 0 + # num_runs threaded into the InferenceConfig for the single request. + assert observed_num_runs["value"] == [2] + # The two per-run results collapse into a single aggregated case that passes. + assert "Tests passed: 1" in result.output + assert "Tests failed: 0" in result.output + + +def test_cli_eval_with_num_runs_prints_details_per_run( + mock_get_root_agent, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +): + app_name = "test_app_num_runs_details" + eval_set_id = "test_eval_set_num_runs_details" + agent_path = tmp_path / app_name + agent_path.mkdir() + (agent_path / "__init__.py").touch() + + eval_sets_manager = LocalEvalSetsManager(agents_dir=str(tmp_path)) + eval_sets_manager.create_eval_set(app_name=app_name, eval_set_id=eval_set_id) + eval_sets_manager.add_eval_case( + app_name=app_name, + eval_set_id=eval_set_id, + eval_case=EvalCase(eval_id="case1", conversation=[]), + ) + + async def _fake_collect_inferences(inference_requests, eval_service): + del inference_requests, eval_service + return [] + + async def _fake_collect_eval_results( + inference_results, eval_service, eval_metrics + ): + del inference_results, eval_service, eval_metrics + return [ + EvalCaseResult( + eval_set_id=eval_set_id, + eval_id="case1", + final_eval_status=EvalStatus.PASSED, + overall_eval_metric_results=[ + EvalMetricResult( + metric_name="response_match_score", + threshold=0.8, + score=1.0, + eval_status=EvalStatus.PASSED, + ) + ], + eval_metric_result_per_invocation=[], + session_id="session_1", + ), + EvalCaseResult( + eval_set_id=eval_set_id, + eval_id="case1", + final_eval_status=EvalStatus.FAILED, + overall_eval_metric_results=[ + EvalMetricResult( + metric_name="response_match_score", + threshold=0.8, + score=0.6, + eval_status=EvalStatus.FAILED, + ) + ], + eval_metric_result_per_invocation=[], + session_id="session_2", + ), + ] + + pretty_print_calls = _Recorder() + monkeypatch.setattr( + "google.adk.cli.cli_eval._collect_inferences", _fake_collect_inferences + ) + monkeypatch.setattr( + "google.adk.cli.cli_eval._collect_eval_results", + _fake_collect_eval_results, + ) + monkeypatch.setattr( + "google.adk.cli.cli_eval.pretty_print_eval_result", pretty_print_calls + ) + + result = CliRunner().invoke( + cli_tools_click.main, + [ + "eval", + str(agent_path), + eval_set_id, + "--num_runs", + "2", + "--print_detailed_results", + ], + ) + + assert result.exit_code == 0 + assert len(pretty_print_calls.calls) == 2 + + def test_cli_create_eval_set(tmp_path: Path): app_name = "test_app" eval_set_id = "test_eval_set" diff --git a/tests/unittests/evaluation/test_eval_case_result_aggregator.py b/tests/unittests/evaluation/test_eval_case_result_aggregator.py new file mode 100644 index 00000000000..152fb858a2f --- /dev/null +++ b/tests/unittests/evaluation/test_eval_case_result_aggregator.py @@ -0,0 +1,210 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from google.adk.evaluation._eval_case_result_aggregator import aggregate_eval_case_results +from google.adk.evaluation.base_eval_service import AggregationStrategy +from google.adk.evaluation.eval_case import Invocation +from google.adk.evaluation.eval_metrics import EvalMetricResult +from google.adk.evaluation.eval_metrics import EvalMetricResultPerInvocation +from google.adk.evaluation.eval_metrics import EvalStatus +from google.adk.evaluation.eval_result import EvalCaseResult +from google.genai import types as genai_types +import pytest + +_METRIC = "response_match_score" +_THRESHOLD = 0.7 + + +def _invocation(score: float) -> EvalMetricResultPerInvocation: + metric_result = EvalMetricResult( + metric_name=_METRIC, + threshold=_THRESHOLD, + score=score, + eval_status=EvalStatus.PASSED + if score >= _THRESHOLD + else EvalStatus.FAILED, + ) + return EvalMetricResultPerInvocation( + actual_invocation=Invocation( + user_content=genai_types.Content(parts=[genai_types.Part(text="hi")]) + ), + eval_metric_results=[metric_result], + ) + + +def _run( + *, + eval_set_id: str, + eval_id: str, + invocation_scores: list[float], + session_id: str, + final_eval_status: EvalStatus = EvalStatus.PASSED, +) -> EvalCaseResult: + per_invocation = [_invocation(score) for score in invocation_scores] + # `overall_eval_metric_results` is deliberately left empty: the aggregator + # must derive the aggregate from per-invocation results, not from a run's + # already-overall score. + return EvalCaseResult( + eval_set_id=eval_set_id, + eval_id=eval_id, + final_eval_status=final_eval_status, + overall_eval_metric_results=[], + eval_metric_result_per_invocation=per_invocation, + session_id=session_id, + ) + + +def test_mean_is_taken_over_invocations_not_over_runs(): + # Run A: 1 invocation @ 0.0. Run B: 3 invocations @ 1.0. + # Mean over invocations: (0 + 1 + 1 + 1) / 4 = 0.75 -> PASSED (>= 0.7). + # Mean of per-run means would be (0.0 + 1.0) / 2 = 0.5 -> FAILED. + results = [ + _run( + eval_set_id="set1", + eval_id="case1", + invocation_scores=[0.0], + session_id="s1", + ), + _run( + eval_set_id="set1", + eval_id="case1", + invocation_scores=[1.0, 1.0, 1.0], + session_id="s2", + ), + ] + + aggregated = aggregate_eval_case_results( + results, + aggregation_strategy=AggregationStrategy.MEAN_OVER_INVOCATIONS, + ) + + assert len(aggregated) == 1 + metric_result = aggregated[0].overall_eval_metric_results[0] + assert metric_result.score == pytest.approx(0.75) + assert metric_result.eval_status == EvalStatus.PASSED + assert aggregated[0].final_eval_status == EvalStatus.PASSED + + +def test_groups_by_eval_set_id_and_eval_id_and_sorts(): + results = [ + _run( + eval_set_id="set2", + eval_id="caseB", + invocation_scores=[1.0], + session_id="s1", + ), + _run( + eval_set_id="set1", + eval_id="caseA", + invocation_scores=[1.0], + session_id="s2", + ), + _run( + eval_set_id="set1", + eval_id="caseA", + invocation_scores=[1.0], + session_id="s3", + ), + ] + + aggregated = aggregate_eval_case_results(results) + + assert [(r.eval_set_id, r.eval_id) for r in aggregated] == [ + ("set1", "caseA"), + ("set2", "caseB"), + ] + + +def test_retains_all_runs_invocations(): + results = [ + _run( + eval_set_id="set1", + eval_id="case1", + invocation_scores=[1.0, 1.0], + session_id="s1", + ), + _run( + eval_set_id="set1", + eval_id="case1", + invocation_scores=[1.0], + session_id="s2", + ), + ] + + aggregated = aggregate_eval_case_results(results) + + assert len(aggregated[0].eval_metric_result_per_invocation) == 3 + + +def test_hard_failed_run_forces_failure(): + # One run passes; another crashed (FAILED, no metric results). The crashed + # run contributes no scores, so honor its failure explicitly. + passing = _run( + eval_set_id="set1", + eval_id="case1", + invocation_scores=[1.0], + session_id="s1", + ) + crashed = EvalCaseResult( + eval_set_id="set1", + eval_id="case1", + final_eval_status=EvalStatus.FAILED, + overall_eval_metric_results=[], + eval_metric_result_per_invocation=[], + session_id="s2", + ) + + aggregated = aggregate_eval_case_results([passing, crashed]) + + assert len(aggregated) == 1 + assert aggregated[0].final_eval_status == EvalStatus.FAILED + + +def test_below_threshold_fails(): + results = [ + _run( + eval_set_id="set1", + eval_id="case1", + invocation_scores=[0.0, 0.5], + session_id="s1", + ), + ] + + aggregated = aggregate_eval_case_results(results) + + assert aggregated[0].overall_eval_metric_results[0].score == pytest.approx( + 0.25 + ) + assert aggregated[0].final_eval_status == EvalStatus.FAILED + + +def test_unsupported_strategy_raises(): + results = [ + _run( + eval_set_id="set1", + eval_id="case1", + invocation_scores=[1.0], + session_id="s1", + ), + ] + + class _Other: + pass + + with pytest.raises(ValueError): + aggregate_eval_case_results( + results, aggregation_strategy=_Other() # type: ignore[arg-type] + ) diff --git a/tests/unittests/evaluation/test_local_eval_service.py b/tests/unittests/evaluation/test_local_eval_service.py index 8223adc341d..e695a3a32b1 100644 --- a/tests/unittests/evaluation/test_local_eval_service.py +++ b/tests/unittests/evaluation/test_local_eval_service.py @@ -211,6 +211,42 @@ async def test_perform_inference_success( assert eval_service._perform_inference_single_eval_item.call_count == 2 +@pytest.mark.asyncio +async def test_perform_inference_repeats_each_case_num_runs_times( + eval_service, + dummy_agent, + mock_eval_sets_manager, + mocker, +): + eval_set = EvalSet( + eval_set_id="test_eval_set", + eval_cases=[ + EvalCase(eval_id="case1", conversation=[], session_input=None), + EvalCase(eval_id="case2", conversation=[], session_input=None), + ], + ) + mock_eval_sets_manager.get_eval_set.return_value = eval_set + + mock_inference_result = mocker.MagicMock() + eval_service._perform_inference_single_eval_item = mocker.AsyncMock( + return_value=mock_inference_result + ) + + inference_request = InferenceRequest( + app_name="test_app", + eval_set_id="test_eval_set", + inference_config=InferenceConfig(parallelism=2, num_runs=3), + ) + + results = [] + async for result in eval_service.perform_inference(inference_request): + results.append(result) + + # 2 eval cases, each inferenced 3 times. + assert len(results) == 6 + assert eval_service._perform_inference_single_eval_item.call_count == 6 + + @pytest.mark.asyncio async def test_perform_inference_with_case_ids( eval_service,